Skip to content

fix(cost): preserve OpenAI-compatible cache hit usage #5774

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Tanisha-Katara wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(cost): preserve OpenAI-compatible cache hit usage #5774

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
161 changes: 136 additions & 25 deletions packages/__tests__/cost/usageProcessor.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { OpenAIUsageProcessor } from "@helicone-package/cost/usage/openAIUsagePr
import { AnthropicUsageProcessor } from "@helicone-package/cost/usage/anthropicUsageProcessor";
import { GroqUsageProcessor } from "@helicone-package/cost/usage/groqUsageProcessor";
import { XAIUsageProcessor } from "@helicone-package/cost/usage/xaiUsageProcessor";
import { OpenRouterUsageProcessor } from "@helicone-package/cost/usage/openRouterUsageProcessor";
import { DeepSeekUsageProcessor } from "@helicone-package/cost/usage/deepseekUsageProcessor";
import { GoogleUsageProcessor } from "@helicone-package/cost/usage/googleUsageProcessor";
import {
Expand Down Expand Up @@ -97,7 +98,7 @@ describe("OpenAIUsageProcessor", () => {
it("should parse real GPT-4o response with cached tokens", async () => {
const responseData = fs.readFileSync(
path.join(__dirname, "testData", "gpt4o-response-cached.snapshot"),
"utf-8"
"utf-8",
);

const result = await processor.parse({
Expand All @@ -121,7 +122,7 @@ describe("OpenAIUsageProcessor", () => {
it("should parse real GPT-4o stream response", async () => {
const streamData = fs.readFileSync(
path.join(__dirname, "testData", "gpt4o-stream-response.snapshot"),
"utf-8"
"utf-8",
);

const result = await processor.parse({
Expand All @@ -137,21 +138,100 @@ describe("OpenAIUsageProcessor", () => {
});
});

it("should parse OpenAI-compatible top-level cache hit tokens", async () => {
const result = await processor.parse({
responseBody: JSON.stringify({
usage: {
prompt_tokens: 100,
completion_tokens: 20,
prompt_cache_hit_tokens: 40,
prompt_cache_miss_tokens: 60,
},
}),
isStream: false,
model: "deepseek/deepseek-chat",
});

expect(result.error).toBeNull();
expect(result.data).toEqual({
input: 60,
output: 20,
cacheDetails: {
cachedInput: 40,
write5m: 0,
write1h: 0,
},
});
});

it("should derive prompt tokens from top-level cache hit and miss tokens", async () => {
const result = await processor.parse({
responseBody: JSON.stringify({
usage: {
completion_tokens: 20,
prompt_cache_hit_tokens: 40,
prompt_cache_miss_tokens: 60,
},
}),
isStream: false,
model: "deepseek/deepseek-chat",
});

expect(result.error).toBeNull();
expect(result.data).toEqual({
input: 60,
output: 20,
cacheDetails: {
cachedInput: 40,
write5m: 0,
write1h: 0,
},
});
});

it("should prefer nested cached tokens over top-level cache hit tokens", async () => {
const result = await processor.parse({
responseBody: JSON.stringify({
usage: {
prompt_tokens: 100,
completion_tokens: 20,
prompt_cache_hit_tokens: 40,
prompt_tokens_details: {
cached_tokens: 25,
},
},
}),
isStream: false,
model: "deepseek/deepseek-chat",
});

expect(result.error).toBeNull();
expect(result.data).toEqual({
input: 75,
output: 20,
cacheDetails: {
cachedInput: 25,
write5m: 0,
write1h: 0,
},
});
});

it("usage processing snapshot", async () => {
const testCases = [
{
name: "cached-response",
data: fs.readFileSync(
path.join(__dirname, "testData", "gpt4o-response-cached.snapshot"),
"utf-8"
"utf-8",
),
isStream: false,
},
{
name: "stream-response",
data: fs.readFileSync(
path.join(__dirname, "testData", "gpt4o-stream-response.snapshot"),
"utf-8"
"utf-8",
),
isStream: true,
},
Expand All @@ -172,13 +252,46 @@ describe("OpenAIUsageProcessor", () => {
});
});

describe("OpenRouterUsageProcessor", () => {
const processor = new OpenRouterUsageProcessor();

it("should preserve top-level cache hit tokens for analytics", async () => {
const result = await processor.parse({
responseBody: JSON.stringify({
usage: {
prompt_tokens: 100,
completion_tokens: 20,
prompt_cache_hit_tokens: 40,
prompt_cache_miss_tokens: 60,
cost: 0.001,
},
}),
isStream: false,
model: "deepseek/deepseek-chat",
});

expect(result.error).toBeNull();
expect(result.data).toEqual({
input: 60,
output: 20,
cost: 0.001,
cost_details: undefined,
provider: undefined,
is_byok: undefined,
cacheDetails: {
cachedInput: 40,
},
});
});
});

describe("Azure Usage Processing", () => {
const processor = new OpenAIUsageProcessor(); // Azure uses OpenAI processor

it("should parse Azure regular response", async () => {
const responseData = fs.readFileSync(
path.join(__dirname, "testData", "azure-response.snapshot"),
"utf-8"
"utf-8",
);

const result = await processor.parse({
Expand All @@ -197,7 +310,7 @@ describe("Azure Usage Processing", () => {
it("should parse Azure stream response", async () => {
const streamData = fs.readFileSync(
path.join(__dirname, "testData", "azure-stream-response.snapshot"),
"utf-8"
"utf-8",
);

const result = await processor.parse({
Expand All @@ -220,15 +333,15 @@ describe("Azure Usage Processing", () => {
name: "azure-response",
data: fs.readFileSync(
path.join(__dirname, "testData", "azure-response.snapshot"),
"utf-8"
"utf-8",
),
isStream: false,
},
{
name: "azure-stream-response",
data: fs.readFileSync(
path.join(__dirname, "testData", "azure-stream-response.snapshot"),
"utf-8"
"utf-8",
),
isStream: true,
},
Expand All @@ -255,7 +368,7 @@ describe("AnthropicUsageProcessor", () => {
it("should parse Anthropic response with cache details", async () => {
const responseData = fs.readFileSync(
path.join(__dirname, "testData", "anthropic-response.snapshot"),
"utf-8"
"utf-8",
);

const result = await processor.parse({
Expand All @@ -278,7 +391,7 @@ describe("AnthropicUsageProcessor", () => {
it("should parse Anthropic stream response", async () => {
const streamData = fs.readFileSync(
path.join(__dirname, "testData", "anthropic-stream-response.snapshot"),
"utf-8"
"utf-8",
);

const result = await processor.parse({
Expand All @@ -304,7 +417,7 @@ describe("AnthropicUsageProcessor", () => {
name: "anthropic-response",
data: fs.readFileSync(
path.join(__dirname, "testData", "anthropic-response.snapshot"),
"utf-8"
"utf-8",
),
isStream: false,
},
Expand All @@ -314,9 +427,9 @@ describe("AnthropicUsageProcessor", () => {
path.join(
__dirname,
"testData",
"anthropic-stream-response.snapshot"
"anthropic-stream-response.snapshot",
),
"utf-8"
"utf-8",
),
isStream: true,
},
Expand All @@ -343,7 +456,7 @@ describe("XAI/Grok specific features", () => {
it("should parse XAI response with web search", async () => {
const xaiResponse = fs.readFileSync(
path.join(__dirname, "testData", "xai-response-websearch.snapshot"),
"utf-8"
"utf-8",
);

const result = await xaiProcessor.parse({
Expand All @@ -366,7 +479,7 @@ describe("XAI/Grok specific features", () => {
it("should parse XAI response with reasoning tokens", async () => {
const xaiResponse = fs.readFileSync(
path.join(__dirname, "testData", "xai-response-reasoning.snapshot"),
"utf-8"
"utf-8",
);

const result = await xaiProcessor.parse({
Expand All @@ -391,7 +504,7 @@ describe("XAI/Grok specific features", () => {
it("should parse XAI stream response with web search", async () => {
const streamData = fs.readFileSync(
path.join(__dirname, "testData", "xai-stream-response.snapshot"),
"utf-8"
"utf-8",
);

const result = await xaiProcessor.parse({
Expand All @@ -418,7 +531,7 @@ describe("Groq specific features", () => {
it("should parse Groq non-streaming response", async () => {
const groqResponse = fs.readFileSync(
path.join(__dirname, "testData", "groq-response.snapshot"),
"utf-8"
"utf-8",
);

const result = await groqProcessor.parse({
Expand All @@ -437,7 +550,7 @@ describe("Groq specific features", () => {
it("should parse Groq streaming response with usage in x_groq", async () => {
const streamData = fs.readFileSync(
path.join(__dirname, "testData", "groq-stream-response.snapshot"),
"utf-8"
"utf-8",
);

const result = await groqProcessor.parse({
Expand All @@ -459,7 +572,7 @@ describe("Groq specific features", () => {
it("should parse real DeepSeek non-streaming response", async () => {
const responseData = fs.readFileSync(
path.join(__dirname, "testData", "deepseek-non-stream.snapshot"),
"utf-8"
"utf-8",
);

const result = await deepseekProcessor.parse({
Expand All @@ -478,7 +591,7 @@ describe("Groq specific features", () => {
it("should parse DeepSeek response with cache hits", async () => {
const responseData = fs.readFileSync(
path.join(__dirname, "testData", "deepseek-cached.snapshot"),
"utf-8"
"utf-8",
);

const result = await deepseekProcessor.parse({
Expand All @@ -500,7 +613,7 @@ describe("Groq specific features", () => {
it("should parse DeepSeek reasoner response with thinking tokens", async () => {
const responseData = fs.readFileSync(
path.join(__dirname, "testData", "deepseek-reasoner.snapshot"),
"utf-8"
"utf-8",
);

const result = await deepseekProcessor.parse({
Expand All @@ -523,7 +636,7 @@ describe("Groq specific features", () => {
it("should parse DeepSeek streaming response", async () => {
const streamData = fs.readFileSync(
path.join(__dirname, "testData", "deepseek-stream.snapshot"),
"utf-8"
"utf-8",
);

const result = await deepseekProcessor.parse({
Expand Down Expand Up @@ -677,9 +790,7 @@ describe("VertexUsageProcessor", () => {
{ modality: "TEXT", tokenCount: 6 },
{ modality: "IMAGE", tokenCount: 8 },
],
candidatesTokensDetails: [
{ modality: "TEXT", tokenCount: 19 },
],
candidatesTokensDetails: [{ modality: "TEXT", tokenCount: 19 }],
},
modelVersion: "gemini-2.5-flash",
responseId: "abc",
Expand Down
Loading