From 15d90f81c5b1c03b4e95ebe3d460a775b56520b6 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 14 Sep 2026 20:11:42 -0400 Subject: [PATCH] Keep default parsers when only some are overridden readRowGroup built columnDecoder with the merged parsers first and then spread the raw options on top, so whenever a caller passed a parsers object it overwrote the merge and every default parser was lost. Reading a file with a date, timestamp, UUID, geometry or JSON column then threw "parser is not a function". Moving the merge below the two spreads makes a partial parsers object override only the named parsers. The public parsers option is now Partial, since a partial object was always the intended input but the declared type demanded all nine. The three functions that merge the defaults now keep the merged set in a local, because a reassigned Partial parameter stays Partial for TypeScript. prefetchPageIndexes only needed its JSDoc widened, since it forwards parsers straight to readColumnIndex. --- src/indexes.js | 8 ++++---- src/metadata.js | 4 ++-- src/plan.js | 2 +- src/rowgroup.js | 9 +++++---- src/types.d.ts | 4 ++-- test/read.test.js | 10 ++++++++++ 6 files changed, 24 insertions(+), 13 deletions(-) diff --git a/src/indexes.js b/src/indexes.js index e12509ea..98f44863 100644 --- a/src/indexes.js +++ b/src/indexes.js @@ -10,17 +10,17 @@ import { deserializeTCompactProtocol } from './thrift.js' /** * @param {DataReader} reader * @param {SchemaElement} schema - * @param {ParquetParsers | undefined} parsers + * @param {Partial | undefined} parsers * @returns {ColumnIndex} */ export function readColumnIndex(reader, schema, parsers = undefined) { - parsers = { ...DEFAULT_PARSERS, ...parsers } + const allParsers = { ...DEFAULT_PARSERS, ...parsers } const thrift = deserializeTCompactProtocol(reader) return { null_pages: thrift.field_1, - min_values: thrift.field_2.map((/** @type {any} */ m) => convertMetadata(m, schema, parsers)), - max_values: thrift.field_3.map((/** @type {any} */ m) => convertMetadata(m, schema, parsers)), + min_values: thrift.field_2.map((/** @type {any} */ m) => convertMetadata(m, schema, allParsers)), + max_values: thrift.field_3.map((/** @type {any} */ m) => convertMetadata(m, schema, allParsers)), boundary_order: BoundaryOrders[thrift.field_4], null_counts: thrift.field_5, repetition_level_histograms: thrift.field_6, diff --git a/src/metadata.js b/src/metadata.js index bcffc578..884b83a4 100644 --- a/src/metadata.js +++ b/src/metadata.js @@ -88,7 +88,7 @@ export function parquetMetadata(arrayBuffer, { parsers, geoparquet = true } = {} const view = new DataView(arrayBuffer) // Use default parsers if not given - parsers = { ...DEFAULT_PARSERS, ...parsers } + const allParsers = { ...DEFAULT_PARSERS, ...parsers } // Validate footer magic number "PAR1" if (view.byteLength < 8) { @@ -148,7 +148,7 @@ export function parquetMetadata(arrayBuffer, { parsers, geoparquet = true } = {} data_page_offset: column.field_3.field_9, index_page_offset: column.field_3.field_10, dictionary_page_offset: column.field_3.field_11, - statistics: convertStats(column.field_3.field_12, columnSchema[columnIndex], parsers), + statistics: convertStats(column.field_3.field_12, columnSchema[columnIndex], allParsers), encoding_stats: column.field_3.field_13?.map((/** @type {any} */ encodingStat) => ({ page_type: PageTypes[encodingStat.field_1], encoding: Encodings[encodingStat.field_2], diff --git a/src/plan.js b/src/plan.js index a32de92d..c737ae4f 100644 --- a/src/plan.js +++ b/src/plan.js @@ -295,7 +295,7 @@ export async function prefetchBloomFilters({ file, metadata, filter, filterStric * @param {string[]} [options.columns] * @param {Record[]} [options.bloomFiltersByGroup] * @param {Record} [options.schemaElements] - * @param {ParquetParsers} [options.parsers] + * @param {Partial} [options.parsers] * @returns {Promise<{pageRangesByGroup: (PageRanges | undefined)[], pageLocationsByGroup: Record[]}>} */ export async function prefetchPageIndexes({ file, metadata, filter, filterStrict = true, rowStart = 0, rowEnd = Infinity, columns, bloomFiltersByGroup, schemaElements, parsers }) { diff --git a/src/rowgroup.js b/src/rowgroup.js index f5d1dc31..0bcded32 100644 --- a/src/rowgroup.js +++ b/src/rowgroup.js @@ -29,9 +29,10 @@ export function readRowGroup(options, { metadata }, groupPlan) { pathInSchema, element: schemaPath[schemaPath.length - 1].element, schemaPath, - parsers: { ...DEFAULT_PARSERS, ...options.parsers }, ...options, ...chunk.columnMetadata, + // merge after options, so a partial parsers object keeps the defaults + parsers: { ...DEFAULT_PARSERS, ...options.parsers }, } const { startByte, endByte } = chunk.range @@ -221,12 +222,12 @@ export async function asyncGroupToRows({ asyncColumns }, selectStart, selectEnd, * * @param {AsyncRowGroup} asyncRowGroup * @param {SchemaTree} schemaTree - * @param {ParquetParsers} [parsers] + * @param {Partial} [parsers] * @returns {AsyncRowGroup} */ export function assembleAsync(asyncRowGroup, schemaTree, parsers) { const { asyncColumns } = asyncRowGroup - parsers = { ...DEFAULT_PARSERS, ...parsers } + const allParsers = { ...DEFAULT_PARSERS, ...parsers } /** @type {AsyncColumn[]} */ const assembled = [] for (const child of schemaTree.children) { @@ -256,7 +257,7 @@ export function assembleAsync(asyncRowGroup, schemaTree, parsers) { ) } // assemble the column - assembleNested(subcolumnData, child, parsers) + assembleNested(subcolumnData, child, allParsers) const assembled = subcolumnData.get(child.element.name) if (!assembled) throw new Error('parquet column data not assembled') return { data: [assembled], skipped } diff --git a/src/types.d.ts b/src/types.d.ts index e2b72776..b3f05132 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -21,7 +21,7 @@ export interface ParquetParsers { * Parquet Metadata options for metadata parsing */ export interface MetadataOptions { - parsers?: ParquetParsers // custom parsers to decode advanced types + parsers?: Partial // custom parsers to decode advanced types, merged over the defaults geoparquet?: boolean // parse geoparquet metadata and set logical type to geometry/geography for geospatial columns (default true) } @@ -41,7 +41,7 @@ export interface BaseParquetReadOptions { onPage?: (chunk: SubColumnData) => void // called when a data page is parsed. pages may contain data outside the requested range. compressors?: Compressors // custom decompressors utf8?: boolean // decode byte arrays as utf8 strings (default true) - parsers?: ParquetParsers // custom parsers to decode advanced types + parsers?: Partial // custom parsers to decode advanced types, merged over the defaults geoparquet?: boolean // parse geoparquet metadata and set logical type to geometry/geography for geospatial columns (default true) useOffsetIndex?: boolean // use offset index to limit column chunk reads when available (default false) useBloomFilters?: boolean // fetch bloom filters to enable row-group skipping on $eq/$in predicates (default false) diff --git a/test/read.test.js b/test/read.test.js index 576b0e65..73a705b4 100644 --- a/test/read.test.js +++ b/test/read.test.js @@ -452,6 +452,16 @@ describe('parquetRead', () => { expect(counting.bytes).toBe(14334) }) + it('keeps default parsers for types a custom parser does not override', async () => { + const file = await asyncBufferFromFile('test/files/duckdb4442.parquet') + const rows = await parquetReadObjects({ + file, + parsers: { stringFromBytes: () => 'custom' }, + }) + expect(rows[0].call_type).toBe('custom') + expect(rows[0].call_date).toEqual(new Date('2011-10-06T22:21:49.580Z')) + }) + it('filter rows with parquetRead', async () => { const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet') await parquetRead({