Skip to content
Open
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
168 changes: 154 additions & 14 deletions frontend/src/components/data-table/charts/__tests__/chart-panel.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import { render, waitFor } from "@testing-library/react";
import { Tooltip } from "radix-ui";
import type { ComponentProps } from "react";
import { beforeAll, describe, expect, it, vi } from "vitest";
import { SetupMocks } from "@/__mocks__/common";
import { LazyVegaEmbed } from "@/components/charts/lazy";
Expand All @@ -18,8 +19,58 @@ beforeAll(() => {
});

describe("ChartPanel", () => {
it("keeps dotted CSV columns aligned with the chart encodings", async () => {
vi.spyOn(vegaLoader, "load").mockResolvedValue("a.b,n\n1,1\n2,2\n3,3\n");
it("reloads CSV only when column types change, not their array identity", async () => {
vi.spyOn(vegaLoader, "load").mockResolvedValue("value\n001\n");
const getDataUrl = vi.fn().mockResolvedValue({
data_url: "chart.csv",
format: "csv",
});
const props: ComponentProps<typeof ChartPanel> = {
tableData: [{ value: "001" }],
chartConfig: {
general: {
xColumn: { field: "value", type: "string" },
yColumn: { field: "value", type: "integer", aggregate: NONE_VALUE },
},
},
chartType: ChartType.BAR,
saveChart: vi.fn(),
saveChartType: vi.fn(),
getDataUrl,
isLargeDataset: false,
};
const panel = (
fieldTypes: ComponentProps<typeof ChartPanel>["fieldTypes"],
) => (
<Tooltip.Provider>
<ChartPanel {...props} fieldTypes={fieldTypes} />
</Tooltip.Provider>
);
const expectValues = async (value: string | number) => {
await waitFor(() => {
expect(vi.mocked(LazyVegaEmbed).mock.lastCall?.[0].spec).toEqual(
expect.objectContaining({ data: { values: [{ value }] } }),
);
});
};

const { rerender } = render(panel([["value", ["string", "object"]]]));
await expectValues("001");
expect(getDataUrl).toHaveBeenCalledTimes(1);

rerender(panel([["value", ["string", "object"]]]));
await expectValues("001");
expect(getDataUrl).toHaveBeenCalledTimes(1);

rerender(panel([["value", ["integer", "int64"]]]));
await expectValues(1);
expect(getDataUrl).toHaveBeenCalledTimes(2);
});

it("uses column types to parse numeric CSV values without coercing text", async () => {
vi.spyOn(vegaLoader, "load").mockResolvedValue(
"a.b,n,label,day,timestamp,active,duration\ninf,1,inf,2024-01-01,2024-01-01T12:00:00Z,True,1 days\n-inf,2,001,2024-01-02,2024-01-02T12:00:00Z,False,2 days\n2.5,3,2024-01-03,2024-01-03,2024-01-03T12:00:00Z,True,3 days\n,4,,,,,\n",
);

render(
<Tooltip.Provider>
Expand All @@ -28,11 +79,7 @@ describe("ChartPanel", () => {
chartConfig={{
general: {
xColumn: { field: "a.b", type: "number" },
yColumn: {
field: "n",
type: "number",
aggregate: NONE_VALUE,
},
yColumn: { field: "n", type: "integer", aggregate: NONE_VALUE },
},
}}
chartType={ChartType.BAR}
Expand All @@ -45,6 +92,11 @@ describe("ChartPanel", () => {
fieldTypes={[
["a.b", ["number", "float64"]],
["n", ["integer", "int64"]],
["label", ["string", "object"]],
["day", ["date", "date"]],
["timestamp", ["datetime", "datetime64[ns]"]],
["active", ["boolean", "bool"]],
["duration", ["unknown", "timedelta64[ns]"]],
]}
isLargeDataset={false}
/>
Expand All @@ -56,17 +108,105 @@ describe("ChartPanel", () => {
expect.objectContaining({
data: {
values: [
{ "a.b": 1, n: 1 },
{ "a.b": 2, n: 2 },
{ "a.b": 3, n: 3 },
{
"a.b": Infinity,
n: 1,
label: "inf",
day: new Date("2024-01-01"),
timestamp: new Date("2024-01-01T12:00:00Z"),
active: true,
duration: "1 days",
},
{
"a.b": -Infinity,
n: 2,
label: "001",
day: new Date("2024-01-02"),
timestamp: new Date("2024-01-02T12:00:00Z"),
active: false,
duration: "2 days",
},
{
"a.b": 2.5,
n: 3,
label: "2024-01-03",
day: new Date("2024-01-03"),
timestamp: new Date("2024-01-03T12:00:00Z"),
active: true,
duration: "3 days",
},
{
"a.b": null,
n: 4,
label: null,
day: "",
timestamp: "",
active: null,
duration: "",
},
],
},
encoding: expect.objectContaining({
x: expect.objectContaining({ field: "a\\.b" }),
y: expect.objectContaining({ field: "n" }),
}),
}),
);
});
});

it.each([true, false])(
"keeps dotted CSV columns aligned with the chart encodings (schema: %s)",
async (hasSchema) => {
vi.spyOn(vegaLoader, "load").mockResolvedValue("a.b,n\n1,1\n2,2\n3,3\n");

render(
<Tooltip.Provider>
<ChartPanel
tableData={[{ "a.b": 1, n: 1 }]}
chartConfig={{
general: {
xColumn: { field: "a.b", type: "number" },
yColumn: {
field: "n",
type: "number",
aggregate: NONE_VALUE,
},
},
}}
chartType={ChartType.BAR}
saveChart={vi.fn()}
saveChartType={vi.fn()}
getDataUrl={vi.fn().mockResolvedValue({
data_url: "chart.csv",
format: "csv",
})}
fieldTypes={
hasSchema
? [
["a.b", ["number", "float64"]],
["n", ["integer", "int64"]],
]
: undefined
}
isLargeDataset={false}
/>
</Tooltip.Provider>,
);

await waitFor(() => {
expect(vi.mocked(LazyVegaEmbed).mock.lastCall?.[0].spec).toEqual(
expect.objectContaining({
data: {
values: [
{ "a.b": 1, n: 1 },
{ "a.b": 2, n: 2 },
{ "a.b": 3, n: 3 },
],
},
encoding: expect.objectContaining({
x: expect.objectContaining({ field: "a\\.b" }),
y: expect.objectContaining({ field: "n" }),
}),
}),
);
});
},
);
});
31 changes: 21 additions & 10 deletions frontend/src/components/data-table/charts/charts.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,11 @@ import { Form } from "@/components/ui/form";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import type { CellId } from "@/core/cells/ids";
import { useAsyncData } from "@/hooks/useAsyncData";
import { useDeepCompareMemoize } from "@/hooks/useDeepCompareMemoize";
import { useDebouncedCallback } from "@/hooks/useDebounce";
import type { GetDataUrl } from "@/plugins/impl/DataTablePlugin";
import { vegaLoadData } from "@/plugins/impl/vega/loader";
import { getVegaFieldTypes } from "@/plugins/impl/vega/utils";
import { useTheme } from "@/theme/useTheme";
import { uniqueBy } from "@/utils/arrays";
import { inferFieldTypes } from "../columns";
Expand Down Expand Up @@ -342,17 +344,26 @@ export const ChartPanel: React.FC<{
return response.data_url;
}

const chartData = await vegaLoadData(
response.data_url,
response.format === "arrow"
? { type: "arrow" }
: response.format === "json"
? { type: "json" }
: { type: "csv", parse: "auto" },
);
return chartData;
let format: Parameters<typeof vegaLoadData>[1];
if (response.format === "arrow") {
format = { type: "arrow" };
} else if (response.format === "json") {
format = { type: "json" };
} else {
format = {
type: "csv",
parse: getVegaFieldTypes(
Comment thread
Light2Dark marked this conversation as resolved.
fieldTypes &&
Object.fromEntries(
fieldTypes.map(([name, [type]]) => [name, type]),
),
Comment thread
Light2Dark marked this conversation as resolved.
{ parseDates: true },
),
};
}
return vegaLoadData(response.data_url, format);
// Re-run when the data table changes
}, [tableData, renderLargeCharts]);
}, [tableData, renderLargeCharts, useDeepCompareMemoize(fieldTypes)]);

const formValues = form.watch();

Expand Down
17 changes: 16 additions & 1 deletion frontend/src/plugins/impl/vega/__tests__/utils.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,22 @@
/* Copyright 2026 Marimo. All rights reserved. */

import { describe, expect, it } from "vitest";
import { getContainerWidth } from "../utils";
import { getContainerWidth, getVegaFieldTypes } from "../utils";

describe("getVegaFieldTypes", () => {
it("preserves date strings unless the caller requests date parsing", () => {
expect(getVegaFieldTypes({ day: "date", timestamp: "datetime" })).toEqual({
day: "string",
timestamp: "date",
});
expect(
getVegaFieldTypes(
{ day: "date", timestamp: "datetime" },
{ parseDates: true },
),
).toEqual({ day: "date", timestamp: "date" });
});
});

describe("getContainerWidth", () => {
it('should return "container" when spec width is "container"', () => {
Expand Down
37 changes: 37 additions & 0 deletions frontend/src/plugins/impl/vega/__tests__/vega.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,43 @@ const { ZERO_WIDTH_SPACE, replacePeriodsInColumnNames, uniquifyColumnNames } =
exportedForTesting;

describe("vega loader", () => {
it("parses infinities only in explicitly numeric CSV fields", async () => {
vi.spyOn(vegaLoader, "load").mockResolvedValue(
"value,label,id\ninf,inf,1\n-inf,-inf,2\n+inf,+inf,3\n2.5,002.5,4\n,,5\nNaN,NaN,6\n",
);

const data = await vegaLoadData("chart.csv", {
type: "csv",
parse: { value: "number", label: "string", id: "integer" },
});

expect(data).toEqual([
{ value: Infinity, label: "inf", id: 1 },
{ value: -Infinity, label: "-inf", id: 2 },
{ value: Infinity, label: "+inf", id: 3 },
{ value: 2.5, label: "002.5", id: 4 },
{ value: null, label: null, id: 5 },
{ value: Number.NaN, label: "NaN", id: 6 },
]);
});

it("preserves the table's large integer and infinity handling", async () => {
vi.spyOn(vegaLoader, "load").mockResolvedValue(
"id,value\n9007199254740993,inf\n9007199254740995,-inf\n",
);

const data = await vegaLoadData(
"table.csv",
{ type: "csv", parse: { id: "integer", value: "number" } },
{ handleBigIntAndNumberLike: true },
);

expect(data).toEqual([
{ id: BigInt("9007199254740993"), value: "inf" },
{ id: BigInt("9007199254740995"), value: "-inf" },
]);
});

it("should parse csv data with dates", async () => {
const csvData = `
active,username,id
Expand Down
24 changes: 23 additions & 1 deletion frontend/src/plugins/impl/vega/loader.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,26 @@ const previousNumberParser = typeParsers.number;
const previousDateParser = typeParsers.date;
const previousBooleanParser = typeParsers.boolean;

const NUMBER_MIDDLEWARE: Middleware = () => {
const parseNumber = (value: string) => {
if (value === "inf" || value === "+inf") {
return Number.POSITIVE_INFINITY;
}
if (value === "-inf") {
return Number.NEGATIVE_INFINITY;
}
return previousNumberParser(value);
};

typeParsers.integer = parseNumber;
Comment thread
Light2Dark marked this conversation as resolved.
typeParsers.number = parseNumber;

return () => {
typeParsers.integer = previousIntegerParser;
typeParsers.number = previousNumberParser;
};
};

const BIG_INT_MIDDLEWARE: Middleware = () => {
// Custom parser to:
// - handle BigInt
Expand Down Expand Up @@ -159,6 +179,8 @@ export async function vegaLoadData<T = object>(
const middleware: Middleware[] = [DATE_MIDDLEWARE];
if (handleBigIntAndNumberLike) {
middleware.push(BIG_INT_MIDDLEWARE);
} else {
middleware.push(NUMBER_MIDDLEWARE);
}

let unsubscribes: Unsubscribe[] = [];
Expand Down Expand Up @@ -216,7 +238,7 @@ export async function vegaLoadData<T = object>(

// Apply middleware
unsubscribes = middleware.map((m) => m());
// Always set parse to auto for csv data, to be able to parse dates and floats
// CSV defaults to inference when no explicit column types are available.
const results = isCsv
? // csv -> json
read(csvOrJsonData, {
Expand Down
9 changes: 5 additions & 4 deletions frontend/src/plugins/impl/vega/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,21 +45,22 @@ function toArray<T>(value: T | T[] | undefined): T[] {

export function getVegaFieldTypes(
types: Record<string, DataType> | undefined | null,
{ parseDates = false }: { parseDates?: boolean } = {},
): FieldTypes | "auto" {
if (!types || Object.keys(types).length === 0) {
// If fieldTypes is provided, use it to parse the data
// Otherwise, infer the data types
return "auto";
}
// Convert all 'date' to 'string', because dates don't format back to
// the correct formatting. For example, a date like '2024-01-01' will
// be formatted to '2024-01-01T00:00:00.000Z'.
// Preserve date-only strings for callers such as the data editor.
// Parsing creates Date objects that JSON serializes as full timestamps.
// Charts opt into date parsing for temporal axes.
return Objects.mapValues(types, (type): VegaDataType => {
if (type === "geometry") {
return "string";
}
if (type === "date") {
return "string";
return parseDates ? "date" : "string";
}
if (type === "time") {
return "string";
Expand Down
Loading