Skip to content

[#12337] feat(web): Support ClickHouse data-skipping indexes and engine-aware sort orders validation #12339

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
LauraXia123 wants to merge 7 commits into
base: main
Choose a base branch
from
Open

[#12337] feat(web): Support ClickHouse data-skipping indexes and engine-aware sort orders validation #12339

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
304958b
[#12337] Support ClickHouse data-skipping indexes and engine-aware so…
LauraXia123 Aug 3, 2026
5492f50
[#12337] fix(web): Align ClickHouse engine detection in handleSubmit …
LauraXia123 Aug 3, 2026
05bbd45
[#12337] fix(web): Preserve backend indexType casing when editing Cli…
LauraXia123 Aug 3, 2026
67b2b23
Merge branch 'main' into issue-12337
LauraXia123 Aug 4, 2026
86e3628
fix by review comment
LauraXia123 Aug 4, 2026
0c5425c
fix lint issue
LauraXia123 Aug 4, 2026
4d69ee1
required properties for ClickHouse Distributed engine
LauraXia123 Aug 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
195 changes: 188 additions & 7 deletions web-v2/web/src/app/catalogs/rightContent/CreateTableDialog.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ import {
ColumnWithParamType,
UnsupportColumnType,
autoIncrementInfoMap,
clickHouseMergeTreeEngines,
defaultValueSupported,
dialogContentMaxHeigth,
distributionInfoMap,
Expand Down Expand Up @@ -115,9 +116,22 @@ export default function CreateTableDialog({ ...props }) {
const [form] = Form.useForm()
const values = Form.useWatch([], form)

const getClickHouseEngine = () => {
if (provider !== 'jdbc-clickhouse') return undefined
return form.getFieldValue('engine') || form.getFieldValue('properties')?.find(item => item?.key === 'engine')?.value
}

const isClickHouseDistributedEngine =
provider === 'jdbc-clickhouse' &&
values?.properties?.find(item => item?.key === 'engine')?.value?.toLowerCase?.() === 'distributed'
(values?.engine || values?.properties?.find(item => item?.key === 'engine')?.value)?.toLowerCase?.() === 'distributed'

const clickHouseEngine =
provider === 'jdbc-clickhouse'
? values?.engine || values?.properties?.find(item => item?.key === 'engine')?.value
: undefined

const isClickHouseMergeTreeEngine =
provider === 'jdbc-clickhouse' && clickHouseMergeTreeEngines.includes(clickHouseEngine)
Comment thread
LauraXia123 marked this conversation as resolved.
Outdated
const isColumnsRequired = !isClickHouseDistributedEngine

const defaultValues = {
Expand Down Expand Up @@ -200,7 +214,16 @@ export default function CreateTableDialog({ ...props }) {
if (sortOredsInfo) {
;``
tabs.push({
label: <span className='font-normal text-[rgb(0,0,0,0.88)]'>Sort Orders</span>,
label: (
<span
className={cn('font-normal text-[rgb(0,0,0,0.88)]', {
'before:mr-0.5 before:font-["SimSun"] before:text-[#ff4d4f] before:content-["*"]':
isClickHouseMergeTreeEngine
})}
>
Sort Orders
</span>
),
key: 'sortOrders'
})
}
Comment thread
LauraXia123 marked this conversation as resolved.
Expand All @@ -226,7 +249,15 @@ export default function CreateTableDialog({ ...props }) {
})
}
setTabOptions(tabs)
}, [isColumnsRequired, provider, partitioningInfo, sortOredsInfo, indexesInfo, distributionInfo])
}, [
isColumnsRequired,
isClickHouseMergeTreeEngine,
provider,
partitioningInfo,
sortOredsInfo,
indexesInfo,
distributionInfo
])

useEffect(() => {
scrollRef.current && handScroll()
Expand All @@ -244,6 +275,30 @@ export default function CreateTableDialog({ ...props }) {
}
}, [values?.distribution?.strategy, provider, values?.partitions, values?.sortOrders])

useEffect(() => {
if (!open || editTable || isLoading) {
return
}

if (!isClickHouseMergeTreeEngine) {
if (values?.sortOrders?.length > 0) {
form.setFieldValue('sortOrders', [])
}
form.setFields([{ name: 'sortOrders', errors: [] }])
} else {
// Re-validate sortOrders when switching to MergeTree engine
const sortOrders = form.getFieldValue('sortOrders')
if (!sortOrders?.length) {
form.setFields([
{
name: 'sortOrders',
errors: ['Sort orders are required for MergeTree family engines']
}
])
}
}
}, [open, editTable, isLoading, isClickHouseMergeTreeEngine])

useEffect(() => {
values?.columns?.forEach((col, index) => {
if (col?.autoIncrement) {
Expand Down Expand Up @@ -469,8 +524,18 @@ export default function CreateTableDialog({ ...props }) {
table.indexes.forEach(item => {
const fields = item.fieldNames.map(f => f[0])
form.setFieldValue(['indexes', idxIndex, 'name'], item.name)
form.setFieldValue(['indexes', idxIndex, 'indexType'], capitalizeFirstLetter(item.indexType))
form.setFieldValue(['indexes', idxIndex, 'indexType'], item.indexType)
form.setFieldValue(['indexes', idxIndex, 'fieldName'], fields)

// Populate index properties
if (item.properties) {
if (item.properties.granularity != null) {
form.setFieldValue(['indexes', idxIndex, 'granularity'], Number(item.properties.granularity))
}
if (item.properties.set_max_values != null) {
form.setFieldValue(['indexes', idxIndex, 'setMaxValues'], Number(item.properties.set_max_values))
}
}
idxIndex++
})
}
Expand Down Expand Up @@ -594,9 +659,54 @@ export default function CreateTableDialog({ ...props }) {

const handleSubmit = e => {
e.preventDefault()

const currentEngine = getClickHouseEngine()
const isCurrentMergeTree = provider === 'jdbc-clickhouse' && clickHouseMergeTreeEngines.includes(currentEngine)

// For non-MergeTree ClickHouse engines, clear sortOrders errors before validating
if (sortOredsInfo && !isCurrentMergeTree) {
form.setFields([{ name: 'sortOrders', errors: [] }])
}

form
.validateFields()
.then(async () => {
// Additional check: for MergeTree engines, sortOrders must not be empty
if (sortOredsInfo && isCurrentMergeTree) {
const sortOrders = form.getFieldValue('sortOrders')
if (!sortOrders?.length) {
form.setFields([
{
name: 'sortOrders',
errors: ['Sort orders are required for MergeTree family engines']
}
])

return Promise.reject({ errorFields: [{ name: ['sortOrders'] }] })
}
const columns = form.getFieldValue('columns') || []

const nullableFields = sortOrders
.filter(s => {
if (!s?.fieldName) return false
const col = columns.find(c => c?.name === s.fieldName)

return col && !col?.required
})
.map(s => s.fieldName)
if (nullableFields.length > 0) {
form.setFields([
{
name: 'sortOrders',
errors: [
`Nullable columns cannot be used in ORDER BY for MergeTree engines: ${nullableFields.join(', ')}`
]
}
])

return Promise.reject({ errorFields: [{ name: ['sortOrders'] }] })
}
}
setConfirmLoading(true)

let submitted = false
Expand Down Expand Up @@ -729,11 +839,27 @@ export default function CreateTableDialog({ ...props }) {
}
if (indexesInfo) {
submitData['indexes'] = values.indexes?.map(i => {
return {
const index = {
indexType: i.indexType,
name: i.name,
fieldNames: i.fieldName.map(f => [f])
}

// Build properties for data skipping indexes
const properties = {}
if (i.indexType?.startsWith('data_skipping_')) {
if (i.granularity != null) {
properties['granularity'] = String(i.granularity)
}
if (i.indexType === 'data_skipping_set' && i.setMaxValues != null) {
properties['set_max_values'] = String(i.setMaxValues)
}
}
if (Object.keys(properties).length > 0) {
index.properties = properties
}

return index
})
}
if (
Expand Down Expand Up @@ -1244,23 +1370,28 @@ export default function CreateTableDialog({ ...props }) {
)
}

const isDataSkippingIndex = indexType => indexType?.startsWith('data_skipping_')

const renderTableIndexes = (fields, subOpt) => {
return (
<div className='flex flex-col divide-y divide-solid border-b border-solid'>
<div className='grid grid-cols-5 divide-x divide-solid'>
<div className='grid grid-cols-7 divide-x divide-solid'>
<div className='col-span-1 bg-gray-100 p-1 text-center'>Index Type</div>
<div className='col-span-2 bg-gray-100 p-1 text-center'>Field</div>
<div className='col-span-1 bg-gray-100 p-1 text-center'>Index Name</div>
<div className='col-span-1 bg-gray-100 p-1 text-center'>Granularity</div>
<div className='col-span-1 bg-gray-100 p-1 text-center'>Set Max Values</div>
<div className='col-span-1 bg-gray-100 p-1 text-center'>Action</div>
</div>
{fields.map(subField => (
<div key={subField.name}>
<div className='grid grid-cols-5'>
<div className='grid grid-cols-7'>
<div className='col-span-1 px-2 py-1'>
<Form.Item noStyle name={[subField.name, 'indexType']} label='Index Type'>
<Select
size='small'
className='w-full'
popupMatchSelectWidth={false}
placeholder='Index Type'
disabled={!!editTable}
onChange={value => {
Expand All @@ -1269,6 +1400,10 @@ export default function CreateTableDialog({ ...props }) {
} else {
form.setFieldValue(['indexes', subField.name, 'name'], '')
}

// Clear properties when index type changes
form.setFieldValue(['indexes', subField.name, 'granularity'], undefined)
form.setFieldValue(['indexes', subField.name, 'setMaxValues'], undefined)
}}
>
{(indexesInfo || []).map(type => (
Expand Down Expand Up @@ -1338,6 +1473,52 @@ export default function CreateTableDialog({ ...props }) {
}}
</Form.Item>
</div>
<div className='col-span-1 px-2 py-1'>
<Form.Item
noStyle
shouldUpdate={(prevValues, curValues) =>
prevValues?.indexes?.[subField.name]?.indexType !== curValues?.indexes?.[subField.name]?.indexType
}
>
{({ getFieldValue }) => {
const currentIndexType = getFieldValue(['indexes', subField.name, 'indexType'])
const showGranularity = isDataSkippingIndex(currentIndexType)

if (!showGranularity) {
return <span className='text-gray-300'>-</span>
}

return (
<Form.Item noStyle name={[subField.name, 'granularity']} label='Granularity'>
<InputNumber size='small' className='w-full' placeholder='1' min={1} disabled={!!editTable} />
</Form.Item>
)
}}
</Form.Item>
</div>
<div className='col-span-1 px-2 py-1'>
<Form.Item
noStyle
shouldUpdate={(prevValues, curValues) =>
prevValues?.indexes?.[subField.name]?.indexType !== curValues?.indexes?.[subField.name]?.indexType
}
>
{({ getFieldValue }) => {
const currentIndexType = getFieldValue(['indexes', subField.name, 'indexType'])
const showSetMaxValues = currentIndexType === 'data_skipping_set'

if (!showSetMaxValues) {
return <span className='text-gray-300'>-</span>
}

return (
<Form.Item noStyle name={[subField.name, 'setMaxValues']} label='Set Max Values'>
<InputNumber size='small' className='w-full' placeholder='0' min={0} disabled={!!editTable} />
</Form.Item>
)
}}
</Form.Item>
</div>
<div className='px-2 py-1'>
<Icons.Minus
className={cn('size-4 cursor-pointer text-gray-400 hover:text-defaultPrimary', {
Expand Down
17 changes: 17 additions & 0 deletions web-v2/web/src/app/catalogs/rightContent/entitiesContent/TableDetailsPage.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,7 @@ export default function TableDetailsPage({ ...props }) {
fields: i.fieldNames,
name: i.name,
indexType: i.indexType,
properties: i.properties,
text: `${i.name}(${i.fieldNames.map(v => v.join('.')).join(',')})`
}
})
Expand Down Expand Up @@ -265,6 +266,22 @@ export default function TableDetailsPage({ ...props }) {
)
})}
</Space.Compact>
<Space.Compact direction='vertical' className='divide-y border-gray-100'>
<span className='min-w-20 bg-gray-100 p-1'>Granularity</span>
{indexList?.map((item, idx) => (
<Tooltip title={item.properties?.granularity} key={`granularity-${idx}`}>
<span className='block max-w-20 truncate p-1'>{item.properties?.granularity ?? '-'}</span>
</Tooltip>
))}
</Space.Compact>
<Space.Compact direction='vertical' className='divide-y border-gray-100'>
<span className='min-w-24 bg-gray-100 p-1'>Set Max Values</span>
{indexList?.map((item, idx) => (
<Tooltip title={item.properties?.set_max_values} key={`set-max-values-${idx}`}>
<span className='block max-w-24 truncate p-1'>{item.properties?.set_max_values ?? '-'}</span>
</Tooltip>
))}
</Space.Compact>
</Space.Compact>
)
}
Expand Down
26 changes: 26 additions & 0 deletions web-v2/web/src/config/catalog.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,32 @@ export const tableDefaultProps = {
select: ['lance', 'delta'],
description: 'The format of the table'
}
],
'jdbc-clickhouse': [
{
key: 'engine',
defaultValue: 'MergeTree',
select: [
'MergeTree',
'ReplacingMergeTree',
'SummingMergeTree',
'AggregatingMergeTree',
'CollapsingMergeTree',
'VersionedCollapsingMergeTree',
'GraphiteMergeTree',
'ReplicatedMergeTree',
'ReplicatedReplacingMergeTree',
'ReplicatedSummingMergeTree',
'ReplicatedAggregatingMergeTree',
'ReplicatedCollapsingMergeTree',
'ReplicatedVersionedCollapsingMergeTree',
'ReplicatedGraphiteMergeTree',
'Distributed',
'TinyLog',
'Log',
'StripeLog'
]
Comment thread
LauraXia123 marked this conversation as resolved.
Outdated
}
Comment thread
LauraXia123 marked this conversation as resolved.
]
}

Expand Down
Loading
Loading