docs: add review assistant example

.gitignore

@@ -6,3 +6,5 @@ node_modules/
 .DS_Store
 axiom-*.tgz
 coverage
+.next
+next-env.d.ts

examples/review-assistant/.env.example

@@ -0,0 +1,5 @@
+AXIOM_URL="https://api.axiom.co"
+AXIOM_TOKEN="xaat-******"
+AXIOM_DATASET="product-telemetry-prod"
+AXIOM_EVALS_DATASET="product-evals-prod"
+OPENAI_API_KEY="sk-******"

examples/review-assistant/axiom.config.ts

@@ -0,0 +1,16 @@
+import 'dotenv/config';
+import { defineConfig } from 'axiom/ai/config';
+import { flagSchema } from './src/app-scope';
+
+export default defineConfig({
+  eval: {
+    include: ['**/*.eval.{ts,js,mts,mjs,cts,cjs}'],
+    exclude: [],
+    dataset: process.env.AXIOM_EVALS_DATASET,
+    url: process.env.AXIOM_URL,
+    token: process.env.AXIOM_TOKEN,
+    orgId: process.env.AXIOM_ORG_ID,
+    flagSchema,
+    timeoutMs: 60_000,
+  },
+});

examples/review-assistant/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "review-analysis",
+  "version": "0.1.0",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "eval": "axiom eval",
+    "server": "tsx --watch src/server.ts"
+  },
+  "dependencies": {
+    "@ai-sdk/openai": "^2.0.72",
+    "@hono/node-server": "^1.19.6",
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/exporter-trace-otlp-proto": "^0.203.0",
+    "@opentelemetry/instrumentation": "^0.203.0",
+    "@opentelemetry/instrumentation-http": "^0.203.0",
+    "@opentelemetry/resources": "^2.0.1",
+    "@opentelemetry/sdk-trace-base": "^2.0.1",
+    "@opentelemetry/sdk-trace-node": "^2.0.1",
+    "@opentelemetry/semantic-conventions": "^1.36.0",
+    "ai": "^5.0.102",
+    "axiom": "workspace:*",
+    "dotenv": "^16.4.5",
+    "hono": "^4.10.6",
+    "tsx": "^4.20.4",
+    "zod": "catalog:"
+  },
+  "devDependencies": {
+    "@types/node": "^22",
+    "typescript": "^5"
+  }
+}

examples/review-assistant/src/app-scope.ts

@@ -0,0 +1,20 @@
+import { openai } from '@ai-sdk/openai';
+import { createAppScope } from 'axiom/ai';
+import { z } from 'zod';
+
+type OpenAIModelId = Parameters<typeof openai>[0];
+
+export const flagSchema = z.object({
+  reviewAssistant: z.object({
+    sentiment: z.object({
+      modelId: z.custom<OpenAIModelId>().default('gpt-4o-mini'),
+    }),
+    summarize: z.object({
+      modelId: z.custom<OpenAIModelId>().default('gpt-4o-mini'),
+    }),
+  }),
+});
+
+const { flag, pickFlags } = createAppScope({ flagSchema });
+
+export { flag, pickFlags };

examples/review-assistant/src/capabilities/assistant.eval.ts

@@ -0,0 +1,76 @@
+import { Eval, Scorer } from 'axiom/ai/evals';
+import { pickFlags } from '@/app-scope';
+import { assistant } from './assistant';
+import { reviews } from '@/collections/reviews';
+import { generateObject } from 'ai';
+import { wrapAISDKModel } from 'axiom/ai';
+import { openai } from '@/openai';
+import { z } from 'zod';
+import { ReviewCollection } from '@/schemas';
+
+const scoringGuidelines = `
+Score from 0.0 to 1.0 based on:
+- Captures key sentiment and main points from the original review
+- 20 words or less
+- Clear and readable
+
+Scoring bands:
+1.0 = Perfect summary - accurate, concise, clear
+0.8 = Good - minor issues with brevity or completeness
+0.6 = Acceptable - misses some details or slightly unclear
+0.4 = Poor - significant inaccuracies or far too long
+0.2 = Very poor - mostly wrong or incomprehensible
+0.0 = Complete failure`;
+
+type AssistantOutput = Awaited<ReturnType<typeof assistant>>;
+
+const sentimentAccuracy = Scorer(
+  'sentiment-accuracy',
+  ({ expected, output }: { expected: { sentiment: string }; output: AssistantOutput }) => {
+    return expected.sentiment === output.sentiment;
+  },
+);
+
+const summaryQuality = Scorer(
+  'summary-quality',
+  async ({
+    input,
+    expected,
+    output,
+  }: {
+    input: { review: string };
+    expected: { summary: string };
+    output: AssistantOutput;
+  }) => {
+    const model = wrapAISDKModel(openai('gpt-4o-mini'));
+
+    const { object: evaluation } = await generateObject({
+      model,
+      schema: z.object({
+        score: z.number().min(0).max(1),
+        reasoning: z.string(),
+      }),
+      prompt: `You are scoring a review summary.
+Original review: \`${input.review}\`
+Generated summary: \`${output.summary}\`
+Reference summary: \`${expected.summary}\`
+${scoringGuidelines}
+Return only a number between 0.0 and 1.0.`,
+    });
+
+    return {
+      score: evaluation.score,
+      metadata: { reasoning: evaluation.reasoning },
+    };
+  },
+);
+
+Eval('review-assistant', {
+  capability: 'review-assistant',
+  configFlags: pickFlags('reviewAssistant'),
+  data: reviews,
+  task: async (task) => {
+    return await assistant(task.input);
+  },
+  scorers: [sentimentAccuracy, summaryQuality],
+});

examples/review-assistant/src/capabilities/assistant.ts

@@ -0,0 +1,44 @@
+import { generateObject } from 'ai';
+import { withSpan, wrapAISDKModel } from 'axiom/ai';
+import { openai } from '@/openai';
+import { z } from 'zod';
+import type { UserReview } from '@/schemas';
+import { flag } from '@/app-scope';
+
+export async function assistant(input: UserReview) {
+  return withSpan({ capability: 'review_assistant', step: 'assistant' }, async () => {
+    const [sentiment, summary] = await Promise.all([
+      withSpan({ capability: 'review_assistant', step: 'sentiment' }, async () => {
+        const modelId = flag('reviewAssistant.sentiment.modelId');
+        const model = wrapAISDKModel(openai(modelId));
+
+        const { object: sentiment } = await generateObject({
+          model,
+          output: 'enum',
+          enum: ['positive', 'negative', 'neutral', 'unknown'],
+          prompt: `Classify the sentiment of the following review: \`${input.review}\``,
+        });
+
+        return sentiment;
+      }),
+
+      withSpan({ capability: 'review_assistant', step: 'summarize' }, async () => {
+        const modelId = flag('reviewAssistant.summarize.modelId');
+        const model = wrapAISDKModel(openai(modelId));
+
+        const { object: summary } = await generateObject({
+          model,
+          schema: z.object({ summary: z.string() }),
+          prompt: `Summarize the following review in 20 words or less: \`${input.review}\``,
+        });
+
+        return summary.summary;
+      }),
+    ]);
+
+    return {
+      sentiment,
+      summary,
+    };
+  });
+}