From 304958b5dca7a9460989350885f4f711e463093c Mon Sep 17 00:00:00 2001 From: Qian Xia Date: Mon, 3 Aug 2026 17:54:43 +0800 Subject: [PATCH 1/6] [#12337] Support ClickHouse data-skipping indexes and engine-aware sort orders validation in CreateTableDialog --- .../rightContent/CreateTableDialog.js | 186 +++++++++++++++++- .../entitiesContent/TableDetailsPage.js | 17 ++ web-v2/web/src/config/catalog.js | 26 +++ web-v2/web/src/config/index.js | 24 ++- web-v2/web/src/lib/store/metalakes/index.js | 1 + 5 files changed, 248 insertions(+), 6 deletions(-) diff --git a/web-v2/web/src/app/catalogs/rightContent/CreateTableDialog.js b/web-v2/web/src/app/catalogs/rightContent/CreateTableDialog.js index 30bc3d72f33..72b5147b121 100644 --- a/web-v2/web/src/app/catalogs/rightContent/CreateTableDialog.js +++ b/web-v2/web/src/app/catalogs/rightContent/CreateTableDialog.js @@ -57,6 +57,7 @@ import { ColumnWithParamType, UnsupportColumnType, autoIncrementInfoMap, + clickHouseMergeTreeEngines, defaultValueSupported, dialogContentMaxHeigth, distributionInfoMap, @@ -118,6 +119,14 @@ export default function CreateTableDialog({ ...props }) { const isClickHouseDistributedEngine = provider === 'jdbc-clickhouse' && values?.properties?.find(item => item?.key === 'engine')?.value?.toLowerCase?.() === 'distributed' + + const clickHouseEngine = + provider === 'jdbc-clickhouse' + ? values?.engine || values?.properties?.find(item => item?.key === 'engine')?.value + : undefined + + const isClickHouseMergeTreeEngine = + provider === 'jdbc-clickhouse' && clickHouseMergeTreeEngines.includes(clickHouseEngine) const isColumnsRequired = !isClickHouseDistributedEngine const defaultValues = { @@ -200,7 +209,16 @@ export default function CreateTableDialog({ ...props }) { if (sortOredsInfo) { ;`` tabs.push({ - label: Sort Orders, + label: ( + + Sort Orders + + ), key: 'sortOrders' }) } @@ -226,7 +244,15 @@ export default function CreateTableDialog({ ...props }) { }) } setTabOptions(tabs) - }, [isColumnsRequired, provider, partitioningInfo, sortOredsInfo, indexesInfo, distributionInfo]) + }, [ + isColumnsRequired, + isClickHouseMergeTreeEngine, + provider, + partitioningInfo, + sortOredsInfo, + indexesInfo, + distributionInfo + ]) useEffect(() => { scrollRef.current && handScroll() @@ -244,6 +270,30 @@ export default function CreateTableDialog({ ...props }) { } }, [values?.distribution?.strategy, provider, values?.partitions, values?.sortOrders]) + useEffect(() => { + if (!open || editTable || isLoading) { + return + } + + if (!isClickHouseMergeTreeEngine) { + if (values?.sortOrders?.length > 0) { + form.setFieldValue('sortOrders', []) + } + form.setFields([{ name: 'sortOrders', errors: [] }]) + } else { + // Re-validate sortOrders when switching to MergeTree engine + const sortOrders = form.getFieldValue('sortOrders') + if (!sortOrders?.length) { + form.setFields([ + { + name: 'sortOrders', + errors: ['Sort orders are required for MergeTree family engines'] + } + ]) + } + } + }, [open, editTable, isLoading, isClickHouseMergeTreeEngine]) + useEffect(() => { values?.columns?.forEach((col, index) => { if (col?.autoIncrement) { @@ -471,6 +521,16 @@ export default function CreateTableDialog({ ...props }) { form.setFieldValue(['indexes', idxIndex, 'name'], item.name) form.setFieldValue(['indexes', idxIndex, 'indexType'], capitalizeFirstLetter(item.indexType)) form.setFieldValue(['indexes', idxIndex, 'fieldName'], fields) + + // Populate index properties + if (item.properties) { + if (item.properties.granularity != null) { + form.setFieldValue(['indexes', idxIndex, 'granularity'], Number(item.properties.granularity)) + } + if (item.properties.set_max_values != null) { + form.setFieldValue(['indexes', idxIndex, 'setMaxValues'], Number(item.properties.set_max_values)) + } + } idxIndex++ }) } @@ -594,9 +654,54 @@ export default function CreateTableDialog({ ...props }) { const handleSubmit = e => { e.preventDefault() + + const currentEngine = form.getFieldValue('engine') + const isCurrentMergeTree = provider === 'jdbc-clickhouse' && clickHouseMergeTreeEngines.includes(currentEngine) + + // For non-MergeTree ClickHouse engines, clear sortOrders errors before validating + if (sortOredsInfo && !isCurrentMergeTree) { + form.setFields([{ name: 'sortOrders', errors: [] }]) + } + form .validateFields() .then(async () => { + // Additional check: for MergeTree engines, sortOrders must not be empty + if (sortOredsInfo && isCurrentMergeTree) { + const sortOrders = form.getFieldValue('sortOrders') + if (!sortOrders?.length) { + form.setFields([ + { + name: 'sortOrders', + errors: ['Sort orders are required for MergeTree family engines'] + } + ]) + + return Promise.reject({ errorFields: [{ name: ['sortOrders'] }] }) + } + const columns = form.getFieldValue('columns') || [] + + const nullableFields = sortOrders + .filter(s => { + if (!s?.fieldName) return false + const col = columns.find(c => c?.name === s.fieldName) + + return col && !col?.required + }) + .map(s => s.fieldName) + if (nullableFields.length > 0) { + form.setFields([ + { + name: 'sortOrders', + errors: [ + `Nullable columns cannot be used in ORDER BY for MergeTree engines: ${nullableFields.join(', ')}` + ] + } + ]) + + return Promise.reject({ errorFields: [{ name: ['sortOrders'] }] }) + } + } setConfirmLoading(true) let submitted = false @@ -729,11 +834,27 @@ export default function CreateTableDialog({ ...props }) { } if (indexesInfo) { submitData['indexes'] = values.indexes?.map(i => { - return { + const index = { indexType: i.indexType, name: i.name, fieldNames: i.fieldName.map(f => [f]) } + + // Build properties for data skipping indexes + const properties = {} + if (i.indexType?.startsWith('data_skipping_')) { + if (i.granularity != null) { + properties['granularity'] = String(i.granularity) + } + if (i.indexType === 'data_skipping_set' && i.setMaxValues != null) { + properties['set_max_values'] = String(i.setMaxValues) + } + } + if (Object.keys(properties).length > 0) { + index.properties = properties + } + + return index }) } if ( @@ -1244,23 +1365,28 @@ export default function CreateTableDialog({ ...props }) { ) } + const isDataSkippingIndex = indexType => indexType?.startsWith('data_skipping_') + const renderTableIndexes = (fields, subOpt) => { return (
-
+
Index Type
Field
Index Name
+
Granularity
+
Set Max Values
Action
{fields.map(subField => (
-
+
- + + {prop.selectGroups ? ( + - {prop.select?.map(item => ( - - {item} - - ))} - - ) : ( - + }} + > + {prop.selectGroups.map(group => ( + + {group.options.map(item => ( + + {item} + + ))} + + ))} + + ) : prop.select ? ( + + ) : ( + + )} + + {prop.key === 'engine' && provider === 'jdbc-clickhouse' && ( + { + if (!visible) { + form.setFieldValue(['engine'], prevEngineRef.current) + pendingEngineRef.current = undefined + setEngineConfirmOpen(false) + } + }} + onConfirm={() => { + prevEngineRef.current = pendingEngineRef.current + pendingEngineRef.current = undefined + setEngineConfirmOpen(false) + }} + onCancel={() => { + form.setFieldValue(['engine'], prevEngineRef.current) + pendingEngineRef.current = undefined + setEngineConfirmOpen(false) + }} + okText='Confirm' + cancelText='Cancel' + > +
+ )} - +
) diff --git a/web-v2/web/src/config/catalog.js b/web-v2/web/src/config/catalog.js index 47f17c7b351..c89791b07c8 100644 --- a/web-v2/web/src/config/catalog.js +++ b/web-v2/web/src/config/catalog.js @@ -17,7 +17,7 @@ * under the License. */ -import { clickHouseMergeTreeEngines } from '@/config' +import { clickHouseEngineGroups } from '@/config' export const checkCatalogIcon = ({ type, provider }) => { switch (type) { @@ -230,7 +230,35 @@ export const tableDefaultProps = { { key: 'engine', defaultValue: 'MergeTree', - select: [...clickHouseMergeTreeEngines, 'Distributed', 'TinyLog', 'Log', 'StripeLog'] + selectGroups: clickHouseEngineGroups + }, + { + key: 'cluster-name', + defaultValue: '', + parentField: 'engine', + show: ['Distributed'], + description: 'The cluster name for DDL operations' + }, + { + key: 'cluster-remote-database', + defaultValue: '', + parentField: 'engine', + show: ['Distributed'], + description: 'The remote database name for ClickHouse distributed tables' + }, + { + key: 'cluster-remote-table', + defaultValue: '', + parentField: 'engine', + show: ['Distributed'], + description: 'The remote table name for ClickHouse distributed tables' + }, + { + key: 'cluster-sharding-key', + defaultValue: '', + parentField: 'engine', + show: ['Distributed'], + description: 'The sharding key for ClickHouse distributed tables' } ] } diff --git a/web-v2/web/src/config/index.js b/web-v2/web/src/config/index.js index c6ffed5af62..03d5be03062 100644 --- a/web-v2/web/src/config/index.js +++ b/web-v2/web/src/config/index.js @@ -360,12 +360,45 @@ export const clickHouseMergeTreeEngines = [ 'AggregatingMergeTree', 'CollapsingMergeTree', 'VersionedCollapsingMergeTree', - 'GraphiteMergeTree', - 'ReplicatedMergeTree', - 'ReplicatedReplacingMergeTree', - 'ReplicatedSummingMergeTree', - 'ReplicatedAggregatingMergeTree', - 'ReplicatedCollapsingMergeTree', - 'ReplicatedVersionedCollapsingMergeTree', - 'ReplicatedGraphiteMergeTree' + 'GraphiteMergeTree' +] + +export const clickHouseLogEngines = ['TinyLog', 'StripeLog', 'Log'] + +export const clickHouseIntegrationEngines = [ + 'ODBC', + 'JDBC', + 'MySQL', + 'MongoDB', + 'Redis', + 'HDFS', + 'S3', + 'Kafka', + 'EmbeddedRocksDB', + 'RabbitMQ', + 'PostgreSQL', + 'S3Queue', + 'TimeSeries' +] + +export const clickHouseSpecialEngines = [ + 'Distributed', + 'Dictionary', + 'Merge', + 'File', + 'Null', + 'Set', + 'Join', + 'URL', + 'View', + 'Memory', + 'Buffer', + 'KeeperMap' +] + +export const clickHouseEngineGroups = [ + { label: 'MergeTree Family', options: clickHouseMergeTreeEngines }, + { label: 'Log Family', options: clickHouseLogEngines }, + { label: 'Integration Engines', options: clickHouseIntegrationEngines }, + { label: 'Special Engines', options: clickHouseSpecialEngines } ]