diff --git a/web-v2/web/src/app/catalogs/rightContent/CreateTableDialog.js b/web-v2/web/src/app/catalogs/rightContent/CreateTableDialog.js index 30bc3d72f33..f049c175043 100644 --- a/web-v2/web/src/app/catalogs/rightContent/CreateTableDialog.js +++ b/web-v2/web/src/app/catalogs/rightContent/CreateTableDialog.js @@ -41,7 +41,21 @@ import React, { useContext, useEffect, useRef, useState } from 'react' import { PlusOutlined } from '@ant-design/icons' -import { Button, Flex, Form, Input, InputNumber, Modal, Pagination, Select, Spin, Switch, Tabs, Typography } from 'antd' +import { + Button, + Flex, + Form, + Input, + InputNumber, + Modal, + Pagination, + Popconfirm, + Select, + Spin, + Switch, + Tabs, + Typography +} from 'antd' import { useScrolling } from 'react-use' import { TreeRefContext } from '../page' import Icons from '@/components/Icons' @@ -57,6 +71,7 @@ import { ColumnWithParamType, UnsupportColumnType, autoIncrementInfoMap, + clickHouseMergeTreeEngines, defaultValueSupported, dialogContentMaxHeigth, distributionInfoMap, @@ -100,6 +115,9 @@ export default function CreateTableDialog({ ...props }) { const [bottomShadow, setBottomShadow] = useState(false) const [topShadow, setTopShadow] = useState(false) const [columnTypes, setColumnTypes] = useState([]) + const prevEngineRef = useRef() + const [engineConfirmOpen, setEngineConfirmOpen] = useState(false) + const pendingEngineRef = useRef() const [tabOptions, setTabOptions] = useState([ { @@ -115,17 +133,27 @@ export default function CreateTableDialog({ ...props }) { const [form] = Form.useForm() const values = Form.useWatch([], form) - const isClickHouseDistributedEngine = + const engineFromProperties = values?.properties?.find(item => item?.key?.toLowerCase?.() === 'engine')?.value + const engineFromTopLevel = values?.engine + const engineFromDefaultProps = tableDefaultProps[provider]?.find(item => item?.key === 'engine')?.defaultValue + const effectiveEngine = engineFromTopLevel || engineFromProperties || engineFromDefaultProps + const confirmedEngine = engineConfirmOpen ? prevEngineRef.current : effectiveEngine + + const allowEmptyColumns = + provider === 'jdbc-clickhouse' && String(confirmedEngine || '').toLowerCase() === 'distributed' + const requireOrderByEngines = clickHouseMergeTreeEngines.map(e => e.toLowerCase()) + + const isMergeTreeEngine = provider === 'jdbc-clickhouse' && - values?.properties?.find(item => item?.key === 'engine')?.value?.toLowerCase?.() === 'distributed' - const isColumnsRequired = !isClickHouseDistributedEngine + !!confirmedEngine && + requireOrderByEngines.includes(String(confirmedEngine).toLowerCase()) + const isSortOrdersRequired = isMergeTreeEngine + const isColumnsRequired = !allowEmptyColumns const defaultValues = { name: '', comment: '', - columns: isClickHouseDistributedEngine - ? [] - : [{ id: '', name: '', typeObj: { type: '' }, required: false, comment: '' }], + columns: allowEmptyColumns ? [] : [{ id: '', name: '', typeObj: { type: '' }, required: false, comment: '' }], properties: [] } const supportProperties = getPropInfo(provider).allowAdd @@ -139,22 +167,34 @@ export default function CreateTableDialog({ ...props }) { const getActiveTableDefaultProps = () => { const props = tableDefaultProps[provider] || [] - if (provider !== 'glue') { - return props - } - - const tableFormat = values?.['table-format']?.toLowerCase() return props.filter(prop => { - if (!prop.hide?.length) { - return true + // If the prop has a "show" list, only show it when the parentField value matches + if (prop.show?.length) { + if (prop.parentField === 'engine') { + const currentEngine = form.getFieldValue('engine') + + return prop.show.includes(currentEngine) + } + const parentValue = values?.[prop.parentField]?.toLowerCase() + if (!parentValue) { + return false + } + + return prop.show.map(item => item.toLowerCase()).includes(parentValue) } - if (!tableFormat) { - return false + // If the prop has a "hide" list, hide it when the parentField value matches + if (prop.hide?.length) { + const parentValue = values?.[prop.parentField]?.toLowerCase() + if (!parentValue) { + return false + } + + return !prop.hide.map(item => item.toLowerCase()).includes(parentValue) } - return !prop.hide.map(item => item.toLowerCase()).includes(tableFormat) + return true }) } @@ -197,10 +237,17 @@ export default function CreateTableDialog({ ...props }) { key: 'partitions' }) } - if (sortOredsInfo) { - ;`` + if (sortOredsInfo && isSortOrdersRequired) { tabs.push({ - label: Sort Orders, + label: ( + + Sort Orders + + ), key: 'sortOrders' }) } @@ -226,7 +273,15 @@ export default function CreateTableDialog({ ...props }) { }) } setTabOptions(tabs) - }, [isColumnsRequired, provider, partitioningInfo, sortOredsInfo, indexesInfo, distributionInfo]) + }, [ + allowEmptyColumns, + isSortOrdersRequired, + provider, + partitioningInfo, + sortOredsInfo, + indexesInfo, + distributionInfo + ]) useEffect(() => { scrollRef.current && handScroll() @@ -244,6 +299,29 @@ export default function CreateTableDialog({ ...props }) { } }, [values?.distribution?.strategy, provider, values?.partitions, values?.sortOrders]) + useEffect(() => { + if (provider === 'jdbc-clickhouse') { + const sortOrdersErrors = form.getFieldError('sortOrders') + if (sortOrdersErrors?.length > 0) { + const sortOrders = form.getFieldValue('sortOrders') + const columns = form.getFieldValue('columns') || [] + const hasSortOrders = Array.isArray(sortOrders) && sortOrders.length > 0 + + const hasNullableSortField = + isMergeTreeEngine && + hasSortOrders && + sortOrders.some(s => { + const col = columns.find(c => c?.name === s?.fieldName) + + return col && !col.required + }) + if ((isSortOrdersRequired && hasSortOrders && !hasNullableSortField) || !isSortOrdersRequired) { + form.setFields([{ name: 'sortOrders', errors: [] }]) + } + } + } + }, [confirmedEngine, provider, values?.sortOrders, values?.columns]) + useEffect(() => { values?.columns?.forEach((col, index) => { if (col?.autoIncrement) { @@ -469,14 +547,27 @@ export default function CreateTableDialog({ ...props }) { table.indexes.forEach(item => { const fields = item.fieldNames.map(f => f[0]) form.setFieldValue(['indexes', idxIndex, 'name'], item.name) - form.setFieldValue(['indexes', idxIndex, 'indexType'], capitalizeFirstLetter(item.indexType)) + form.setFieldValue(['indexes', idxIndex, 'indexType'], item.indexType) form.setFieldValue(['indexes', idxIndex, 'fieldName'], fields) + + // Populate index properties + if (item.properties) { + if (item.properties.granularity != null) { + form.setFieldValue(['indexes', idxIndex, 'granularity'], Number(item.properties.granularity)) + } + if (item.properties.set_max_values != null) { + form.setFieldValue(['indexes', idxIndex, 'setMaxValues'], Number(item.properties.set_max_values)) + } + } idxIndex++ }) } let idxProperty = 0 if (table.properties && Object.keys(table.properties).length) { Object.entries(table.properties).forEach(([key, value]) => { + if (key.toLowerCase() === 'engine') { + prevEngineRef.current = value + } form.setFieldValue(['properties', idxProperty, 'key'], key) form.setFieldValue(['properties', idxProperty, 'value'], value) form.setFieldValue(['properties', idxProperty, 'isEdit'], true) @@ -496,8 +587,17 @@ export default function CreateTableDialog({ ...props }) { if (tableDefaultProps[provider]) { tableDefaultProps[provider].forEach(item => { form.setFieldValue(item.key, item.defaultValue) + if (item.key === 'engine') { + prevEngineRef.current = item.defaultValue + } }) } + const currentColumns = form.getFieldValue('columns') || [] + if (allowEmptyColumns) { + form.setFieldValue('columns', []) + } else if (currentColumns.length === 0) { + form.setFieldValue('columns', [{ id: '', name: '', typeObj: { type: '' }, required: false, comment: '' }]) + } } if (provider) { let columnTypes = [...ColumnType, ...ColumnWithParamType, ...ColumnSpesicalType].filter( @@ -534,16 +634,10 @@ export default function CreateTableDialog({ ...props }) { const columns = form.getFieldValue('columns') || [] - if (isClickHouseDistributedEngine && columns.length === 1 && !columns[0]?.name && !columns[0]?.typeObj?.type) { - form.setFieldValue('columns', []) - - return - } - - if (!isClickHouseDistributedEngine && columns.length === 0) { + if (!allowEmptyColumns && columns.length === 0) { form.setFieldValue('columns', [{ id: '', name: '', typeObj: { type: '' }, required: false, comment: '' }]) } - }, [open, editTable, isClickHouseDistributedEngine, form]) + }, [open, editTable, allowEmptyColumns, form]) const getColumnType = typeObj => { const { type } = typeObj @@ -594,6 +688,7 @@ export default function CreateTableDialog({ ...props }) { const handleSubmit = e => { e.preventDefault() + form .validateFields() .then(async () => { @@ -686,7 +781,7 @@ export default function CreateTableDialog({ ...props }) { } }) } - if (sortOredsInfo) { + if (sortOredsInfo && isSortOrdersRequired) { submitData['sortOrders'] = values.sortOrders?.map(s => { const field = { sortTerm: {} @@ -729,11 +824,27 @@ export default function CreateTableDialog({ ...props }) { } if (indexesInfo) { submitData['indexes'] = values.indexes?.map(i => { - return { - indexType: i.indexType, + const index = { + indexType: provider === 'jdbc-clickhouse' ? i.indexType?.toUpperCase() : i.indexType, name: i.name, fieldNames: i.fieldName.map(f => [f]) } + + // Build properties for data skipping indexes + const properties = {} + if (i.indexType?.startsWith('data_skipping_')) { + if (i.granularity != null) { + properties['granularity'] = String(i.granularity) + } + if (i.indexType === 'data_skipping_set' && i.setMaxValues != null) { + properties['set_max_values'] = String(i.setMaxValues) + } + } + if (Object.keys(properties).length > 0) { + index.properties = properties + } + + return index }) } if ( @@ -1244,23 +1355,28 @@ export default function CreateTableDialog({ ...props }) { ) } + const isDataSkippingIndex = indexType => indexType?.startsWith('data_skipping_') + const renderTableIndexes = (fields, subOpt) => { return (
-
+
Index Type
Field
Index Name
+
Granularity
+
Set Max Values
Action
{fields.map(subField => (
-
+
- + + {prop.selectGroups ? ( + - {prop.select?.map(item => ( - - {item} - - ))} - - ) : ( - + }} + > + {prop.selectGroups.map(group => ( + + {group.options.map(item => ( + + {item} + + ))} + + ))} + + ) : prop.select ? ( + + ) : ( + + )} + + {prop.key === 'engine' && provider === 'jdbc-clickhouse' && ( + { + if (!visible) { + form.setFieldValue(['engine'], prevEngineRef.current) + pendingEngineRef.current = undefined + setEngineConfirmOpen(false) + } + }} + onConfirm={() => { + prevEngineRef.current = pendingEngineRef.current + pendingEngineRef.current = undefined + setEngineConfirmOpen(false) + }} + onCancel={() => { + form.setFieldValue(['engine'], prevEngineRef.current) + pendingEngineRef.current = undefined + setEngineConfirmOpen(false) + }} + okText='Confirm' + cancelText='Cancel' + > +
+ )} - +
) diff --git a/web-v2/web/src/app/catalogs/rightContent/entitiesContent/TableDetailsPage.js b/web-v2/web/src/app/catalogs/rightContent/entitiesContent/TableDetailsPage.js index 90ad4a0d2e7..272a422bfda 100644 --- a/web-v2/web/src/app/catalogs/rightContent/entitiesContent/TableDetailsPage.js +++ b/web-v2/web/src/app/catalogs/rightContent/entitiesContent/TableDetailsPage.js @@ -230,6 +230,7 @@ export default function TableDetailsPage({ ...props }) { fields: i.fieldNames, name: i.name, indexType: i.indexType, + properties: i.properties, text: `${i.name}(${i.fieldNames.map(v => v.join('.')).join(',')})` } }) @@ -265,6 +266,22 @@ export default function TableDetailsPage({ ...props }) { ) })} + + Granularity + {indexList?.map((item, idx) => ( + + {item.properties?.granularity ?? '-'} + + ))} + + + Set Max Values + {indexList?.map((item, idx) => ( + + {item.properties?.set_max_values ?? '-'} + + ))} + ) } diff --git a/web-v2/web/src/config/catalog.js b/web-v2/web/src/config/catalog.js index 41c8881a431..c89791b07c8 100644 --- a/web-v2/web/src/config/catalog.js +++ b/web-v2/web/src/config/catalog.js @@ -17,6 +17,8 @@ * under the License. */ +import { clickHouseEngineGroups } from '@/config' + export const checkCatalogIcon = ({ type, provider }) => { switch (type) { case 'relational': @@ -223,6 +225,41 @@ export const tableDefaultProps = { select: ['lance', 'delta'], description: 'The format of the table' } + ], + 'jdbc-clickhouse': [ + { + key: 'engine', + defaultValue: 'MergeTree', + selectGroups: clickHouseEngineGroups + }, + { + key: 'cluster-name', + defaultValue: '', + parentField: 'engine', + show: ['Distributed'], + description: 'The cluster name for DDL operations' + }, + { + key: 'cluster-remote-database', + defaultValue: '', + parentField: 'engine', + show: ['Distributed'], + description: 'The remote database name for ClickHouse distributed tables' + }, + { + key: 'cluster-remote-table', + defaultValue: '', + parentField: 'engine', + show: ['Distributed'], + description: 'The remote table name for ClickHouse distributed tables' + }, + { + key: 'cluster-sharding-key', + defaultValue: '', + parentField: 'engine', + show: ['Distributed'], + description: 'The sharding key for ClickHouse distributed tables' + } ] } diff --git a/web-v2/web/src/config/index.js b/web-v2/web/src/config/index.js index 15c23285180..03d5be03062 100644 --- a/web-v2/web/src/config/index.js +++ b/web-v2/web/src/config/index.js @@ -238,6 +238,11 @@ const tableLevelPropInfoMap = { reserved: [], immutable: ['default-location-name'], allowAdd: true + }, + 'jdbc-clickhouse': { + reserved: [], + immutable: ['engine'], + allowAdd: true } } @@ -328,7 +333,7 @@ export const indexesInfoMap = { 'jdbc-oceanbase': ['primary_key', 'unique_key'], 'jdbc-postgresql': ['primary_key', 'unique_key'], 'lakehouse-paimon': ['primary_key'], - 'jdbc-clickhouse': ['primary_key'] + 'jdbc-clickhouse': ['primary_key', 'data_skipping_minmax', 'data_skipping_bloom_filter', 'data_skipping_set'] } export const autoIncrementInfoMap = { @@ -347,3 +352,53 @@ export const autoIncrementInfoMap = { } export const defaultValueSupported = ['jdbc-doris', 'jdbc-mysql', 'jdbc-oceanbase', 'jdbc-postgresql', 'jdbc-starrocks'] + +export const clickHouseMergeTreeEngines = [ + 'MergeTree', + 'ReplacingMergeTree', + 'SummingMergeTree', + 'AggregatingMergeTree', + 'CollapsingMergeTree', + 'VersionedCollapsingMergeTree', + 'GraphiteMergeTree' +] + +export const clickHouseLogEngines = ['TinyLog', 'StripeLog', 'Log'] + +export const clickHouseIntegrationEngines = [ + 'ODBC', + 'JDBC', + 'MySQL', + 'MongoDB', + 'Redis', + 'HDFS', + 'S3', + 'Kafka', + 'EmbeddedRocksDB', + 'RabbitMQ', + 'PostgreSQL', + 'S3Queue', + 'TimeSeries' +] + +export const clickHouseSpecialEngines = [ + 'Distributed', + 'Dictionary', + 'Merge', + 'File', + 'Null', + 'Set', + 'Join', + 'URL', + 'View', + 'Memory', + 'Buffer', + 'KeeperMap' +] + +export const clickHouseEngineGroups = [ + { label: 'MergeTree Family', options: clickHouseMergeTreeEngines }, + { label: 'Log Family', options: clickHouseLogEngines }, + { label: 'Integration Engines', options: clickHouseIntegrationEngines }, + { label: 'Special Engines', options: clickHouseSpecialEngines } +] diff --git a/web-v2/web/src/lib/store/metalakes/index.js b/web-v2/web/src/lib/store/metalakes/index.js index 55566d69735..acdc60c28a4 100644 --- a/web-v2/web/src/lib/store/metalakes/index.js +++ b/web-v2/web/src/lib/store/metalakes/index.js @@ -1271,6 +1271,7 @@ export const getTableDetails = createAsyncThunk( fields: i.fieldNames, name: i.name, indexType: i.indexType, + properties: i.properties, text: `${i.name}(${i.fieldNames.map(v => v.join('.')).join(',')})` } })