From 0d2b32a6b5ffa2adc876e5ea4bf3547b464f96bd Mon Sep 17 00:00:00 2001 From: Garik Kalashyan Date: Fri, 10 Jul 2026 13:24:48 +0400 Subject: [PATCH 1/6] Add BigLake configuration validation for catalog creation and updates --- .../admin/BigLakeCatalogValidator.java | 281 ++++++++++++++++++ .../service/admin/PolarisAdminService.java | 2 + .../service/admin/PolarisServiceImpl.java | 1 + .../admin/BigLakeCatalogValidatorTest.java | 270 +++++++++++++++++ .../service/admin/ManagementServiceTest.java | 145 +++++++++ 5 files changed, 699 insertions(+) create mode 100644 runtime/service/src/main/java/org/apache/polaris/service/admin/BigLakeCatalogValidator.java create mode 100644 runtime/service/src/test/java/org/apache/polaris/service/admin/BigLakeCatalogValidatorTest.java diff --git a/runtime/service/src/main/java/org/apache/polaris/service/admin/BigLakeCatalogValidator.java b/runtime/service/src/main/java/org/apache/polaris/service/admin/BigLakeCatalogValidator.java new file mode 100644 index 00000000000..0262f3d88ad --- /dev/null +++ b/runtime/service/src/main/java/org/apache/polaris/service/admin/BigLakeCatalogValidator.java @@ -0,0 +1,281 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.admin; + +import com.google.common.base.Strings; +import java.net.URI; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; +import org.apache.polaris.core.admin.model.AuthenticationParameters; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.ExternalCatalog; +import org.apache.polaris.core.admin.model.GcpStorageConfigInfo; +import org.apache.polaris.core.admin.model.IcebergRestConnectionConfigInfo; +import org.apache.polaris.core.admin.model.StorageConfigInfo; +import org.apache.polaris.core.config.FeatureConfiguration; +import org.apache.polaris.core.config.RealmConfig; +import org.apache.polaris.core.storage.StorageUri; + +final class BigLakeCatalogValidator { + private static final String BIGLAKE_HOST = "biglake.googleapis.com"; + private static final String BIGLAKE_PATH = "/iceberg/v1/restcatalog"; + private static final String DEFAULT_BASE_LOCATION_KEY = "default-base-location"; + private static final String QUOTA_PROJECT_HEADER = "header.x-goog-user-project"; + + private static final Pattern GCP_PROJECT_ID_PATTERN = + Pattern.compile("^[a-z][a-z0-9-]{4,28}[a-z0-9]$"); + private static final Pattern GCP_PROJECT_NUMBER_PATTERN = Pattern.compile("^[1-9][0-9]{5,}$"); + private static final Pattern BIGLAKE_RESOURCE_NAME_PATTERN = + Pattern.compile("^projects/[^/\\s]+/locations/[^/\\s]+/catalogs/[^/\\s]+$"); + private static final Pattern BIGLAKE_SIMPLE_CATALOG_PATTERN = + Pattern.compile("^[A-Za-z0-9._-]+$"); + private static final Pattern GCS_SERVICE_ACCOUNT_PATTERN = + Pattern.compile("^[a-z][a-z0-9-]{4,28}[a-z0-9]@[a-z][a-z0-9-]{4,28}[a-z0-9]\\.iam\\.gserviceaccount\\.com$"); + + private static final Set BLOCKED_HEADER_PROPERTIES = + Set.of("header.authorization", "header.proxy-authorization"); + + private BigLakeCatalogValidator() {} + + static void validate(RealmConfig realmConfig, Catalog catalog) { + if (!(catalog instanceof ExternalCatalog externalCatalog)) { + return; + } + + if (!(externalCatalog.getConnectionConfigInfo() instanceof IcebergRestConnectionConfigInfo connectionConfig)) { + return; + } + + if (connectionConfig.getAuthenticationParameters() == null + || connectionConfig.getAuthenticationParameters().getAuthenticationType() + != AuthenticationParameters.AuthenticationTypeEnum.GCP) { + return; + } + + validateBigLakeEndpoint(connectionConfig.getUri()); + validateBigLakeRemoteCatalogName(connectionConfig.getRemoteCatalogName()); + validateBigLakeHeaders(connectionConfig.getProperties()); + validateBigLakeStorageConfiguration(realmConfig, externalCatalog); + } + + private static void validateBigLakeEndpoint(String uriString) { + if (Strings.isNullOrEmpty(uriString)) { + throw new IllegalArgumentException( + "Invalid BigLake connectionConfigInfo.uri: an https:// BigLake endpoint is required."); + } + + URI uri; + try { + uri = URI.create(uriString); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + "Invalid BigLake connectionConfigInfo.uri '" + uriString + "'.", e); + } + + if (!"https".equalsIgnoreCase(uri.getScheme())) { + throw new IllegalArgumentException( + "Invalid BigLake connectionConfigInfo.uri '" + + uriString + + "': BigLake requires an https:// URI."); + } + + if (!BIGLAKE_HOST.equalsIgnoreCase(uri.getHost())) { + throw new IllegalArgumentException( + "Invalid BigLake connectionConfigInfo.uri '" + + uriString + + "': unsupported host '" + + uri.getHost() + + "'. Expected '" + + BIGLAKE_HOST + + "'."); + } + + String normalizedPath = normalizePath(uri.getPath()); + if (!BIGLAKE_PATH.equals(normalizedPath)) { + throw new IllegalArgumentException( + "Invalid BigLake connectionConfigInfo.uri '" + + uriString + + "': unsupported path '" + + uri.getPath() + + "'. Expected '" + + BIGLAKE_PATH + + "'."); + } + + if (uri.getRawQuery() != null || uri.getRawFragment() != null || uri.getPort() != -1) { + throw new IllegalArgumentException( + "Invalid BigLake connectionConfigInfo.uri '" + + uriString + + "': query, fragment, and custom port components are not supported."); + } + } + + private static void validateBigLakeRemoteCatalogName(String remoteCatalogName) { + if (Strings.isNullOrEmpty(remoteCatalogName) || remoteCatalogName.trim().isEmpty()) { + throw new IllegalArgumentException( + "Invalid BigLake connectionConfigInfo.remoteCatalogName: a remote catalog or warehouse identifier is required."); + } + + String trimmedRemoteCatalogName = remoteCatalogName.trim(); + if (trimmedRemoteCatalogName.startsWith("gs://")) { + validateGsLocation("connectionConfigInfo.remoteCatalogName", trimmedRemoteCatalogName); + return; + } + + if (BIGLAKE_RESOURCE_NAME_PATTERN.matcher(trimmedRemoteCatalogName).matches() + || BIGLAKE_SIMPLE_CATALOG_PATTERN.matcher(trimmedRemoteCatalogName).matches()) { + return; + } + + throw new IllegalArgumentException( + "Invalid BigLake connectionConfigInfo.remoteCatalogName '" + + remoteCatalogName + + "': expected a BigLake catalog identifier or gs:// warehouse location."); + } + + private static void validateBigLakeHeaders(Map properties) { + Map headerProperties = properties != null ? properties : Map.of(); + + for (String propertyName : headerProperties.keySet()) { + if (propertyName == null) { + continue; + } + + String normalizedPropertyName = propertyName.toLowerCase(Locale.ROOT); + if (!normalizedPropertyName.startsWith("header.") + || QUOTA_PROJECT_HEADER.equals(normalizedPropertyName)) { + continue; + } + + if (BLOCKED_HEADER_PROPERTIES.contains(normalizedPropertyName)) { + throw new IllegalArgumentException( + "Invalid BigLake connectionConfigInfo.properties entry '" + + propertyName + + "': overriding security-sensitive headers is not allowed."); + } + + throw new IllegalArgumentException( + "Invalid BigLake connectionConfigInfo.properties entry '" + + propertyName + + "': only '" + + QUOTA_PROJECT_HEADER + + "' is supported."); + } + + String quotaProject = headerProperties.get(QUOTA_PROJECT_HEADER); + if (Strings.isNullOrEmpty(quotaProject) || quotaProject.trim().isEmpty()) { + throw new IllegalArgumentException( + "Invalid BigLake connectionConfigInfo.properties entry '" + + QUOTA_PROJECT_HEADER + + "': a quota project is required."); + } + + String trimmedQuotaProject = quotaProject.trim(); + if (!GCP_PROJECT_ID_PATTERN.matcher(trimmedQuotaProject).matches() + && !GCP_PROJECT_NUMBER_PATTERN.matcher(trimmedQuotaProject).matches()) { + throw new IllegalArgumentException( + "Invalid BigLake connectionConfigInfo.properties entry '" + + QUOTA_PROJECT_HEADER + + "': '" + + quotaProject + + "' is not a valid GCP quota project."); + } + } + + private static void validateBigLakeStorageConfiguration( + RealmConfig realmConfig, ExternalCatalog externalCatalog) { + boolean credentialVendingEnabled = + realmConfig.getConfig( + FeatureConfiguration.ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING, + externalCatalog.getProperties().toMap()) + && realmConfig.getConfig( + FeatureConfiguration.ALLOW_FEDERATED_CATALOGS_CREDENTIAL_VENDING, + externalCatalog.getProperties().toMap()); + + StorageConfigInfo storageConfigInfo = externalCatalog.getStorageConfigInfo(); + if (storageConfigInfo == null) { + if (credentialVendingEnabled) { + throw new IllegalArgumentException( + "Invalid BigLake storageConfigInfo: GCS storage configuration is required when credential vending is enabled."); + } + return; + } + + if (storageConfigInfo.getStorageType() != StorageConfigInfo.StorageTypeEnum.GCS + || !(storageConfigInfo instanceof GcpStorageConfigInfo gcpStorageConfigInfo)) { + throw new IllegalArgumentException( + "Invalid BigLake storageConfigInfo.storageType: expected GCS but found " + + storageConfigInfo.getStorageType() + + "."); + } + + String defaultBaseLocation = externalCatalog.getProperties().toMap().get(DEFAULT_BASE_LOCATION_KEY); + validateGsLocation("catalog.properties." + DEFAULT_BASE_LOCATION_KEY, defaultBaseLocation); + + List allowedLocations = gcpStorageConfigInfo.getAllowedLocations(); + if (allowedLocations != null) { + for (int index = 0; index < allowedLocations.size(); index++) { + validateGsLocation("storageConfigInfo.allowedLocations[" + index + "]", allowedLocations.get(index)); + } + } + + String serviceAccount = gcpStorageConfigInfo.getGcsServiceAccount(); + if (!Strings.isNullOrEmpty(serviceAccount) && !GCS_SERVICE_ACCOUNT_PATTERN.matcher(serviceAccount).matches()) { + throw new IllegalArgumentException( + "Invalid BigLake storageConfigInfo.gcsServiceAccount '" + + serviceAccount + + "': expected a Google service account email."); + } + + if (credentialVendingEnabled && Strings.isNullOrEmpty(serviceAccount)) { + throw new IllegalArgumentException( + "Invalid BigLake storageConfigInfo.gcsServiceAccount: a Google service account is required when credential vending is enabled."); + } + } + + private static void validateGsLocation(String fieldName, String location) { + if (Strings.isNullOrEmpty(location) || location.trim().isEmpty()) { + throw new IllegalArgumentException( + "Invalid BigLake " + fieldName + ": a non-empty gs:// location is required."); + } + + StorageUri storageUri; + try { + storageUri = StorageUri.parse(location); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + "Invalid BigLake " + fieldName + " '" + location + "': malformed gs:// location.", e); + } + + if (!"gs".equalsIgnoreCase(storageUri.scheme()) || Strings.isNullOrEmpty(storageUri.authority())) { + throw new IllegalArgumentException( + "Invalid BigLake " + fieldName + " '" + location + "': expected a gs:// location."); + } + } + + private static String normalizePath(String path) { + if (Strings.isNullOrEmpty(path)) { + return ""; + } + return path.endsWith("/") && path.length() > 1 ? path.substring(0, path.length() - 1) : path; + } +} diff --git a/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java b/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java index 3506fd17710..62502981908 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java @@ -1035,6 +1035,8 @@ private void validateUpdateCatalogDiffOrThrow( } CatalogEntity updatedEntity = updateBuilder.build(); + BigLakeCatalogValidator.validate( + realmConfig, updatedEntity.asCatalog(getServiceIdentityProvider())); validateUpdateCatalogDiffOrThrow(currentCatalogEntity, updatedEntity); if (catalogOverlapsWithExistingCatalog(updatedEntity)) { diff --git a/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java b/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java index fe648588b0b..69b9277cf60 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java @@ -171,6 +171,7 @@ private void validateExternalCatalog(Catalog catalog) { if (connectionConfigInfo != null) { validateConnectionConfigInfo(connectionConfigInfo); validateAuthenticationParameters(connectionConfigInfo.getAuthenticationParameters()); + BigLakeCatalogValidator.validate(realmConfig, catalog); } } } diff --git a/runtime/service/src/test/java/org/apache/polaris/service/admin/BigLakeCatalogValidatorTest.java b/runtime/service/src/test/java/org/apache/polaris/service/admin/BigLakeCatalogValidatorTest.java new file mode 100644 index 00000000000..58d8d49d3c8 --- /dev/null +++ b/runtime/service/src/test/java/org/apache/polaris/service/admin/BigLakeCatalogValidatorTest.java @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.admin; + +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.util.List; +import java.util.Map; +import org.apache.polaris.core.admin.model.AuthenticationParameters; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogProperties; +import org.apache.polaris.core.admin.model.ConnectionConfigInfo; +import org.apache.polaris.core.admin.model.ExternalCatalog; +import org.apache.polaris.core.admin.model.GcpAuthenticationParameters; +import org.apache.polaris.core.admin.model.GcpStorageConfigInfo; +import org.apache.polaris.core.admin.model.IcebergRestConnectionConfigInfo; +import org.apache.polaris.core.admin.model.StorageConfigInfo; +import org.apache.polaris.core.config.RealmConfig; +import org.apache.polaris.service.TestServices; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class BigLakeCatalogValidatorTest { + private RealmConfig realmConfig; + + @BeforeEach + void setup() { + realmConfig = TestServices.builder().build().realmConfig(); + } + + @Test + void validBigLakeConfigurationPasses() { + assertThatCode( + () -> + BigLakeCatalogValidator.validate( + realmConfig, + bigLakeCatalog( + "https://biglake.googleapis.com/iceberg/v1/restcatalog", + "my-remote-catalog", + Map.of("header.x-goog-user-project", "my-billing-project"), + true, + "gs://bucket/path/to/data", + validGcsStorage("gs://bucket/path/to/data")))) + .doesNotThrowAnyException(); + } + + @Test + void rejectsNonHttpsEndpoint() { + assertThatThrownBy( + () -> + BigLakeCatalogValidator.validate( + realmConfig, + bigLakeCatalog( + "http://biglake.googleapis.com/iceberg/v1/restcatalog", + "my-remote-catalog", + Map.of("header.x-goog-user-project", "my-billing-project"), + true, + "gs://bucket/path/to/data", + validGcsStorage("gs://bucket/path/to/data")))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("requires an https:// URI"); + } + + @Test + void rejectsUnsupportedEndpointPath() { + assertThatThrownBy( + () -> + BigLakeCatalogValidator.validate( + realmConfig, + bigLakeCatalog( + "https://biglake.googleapis.com/not-biglake", + "my-remote-catalog", + Map.of("header.x-goog-user-project", "my-billing-project"), + true, + "gs://bucket/path/to/data", + validGcsStorage("gs://bucket/path/to/data")))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("unsupported path"); + } + + @Test + void rejectsMissingRemoteCatalogIdentifier() { + assertThatThrownBy( + () -> + BigLakeCatalogValidator.validate( + realmConfig, + bigLakeCatalog( + "https://biglake.googleapis.com/iceberg/v1/restcatalog", + " ", + Map.of("header.x-goog-user-project", "my-billing-project"), + true, + "gs://bucket/path/to/data", + validGcsStorage("gs://bucket/path/to/data")))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("remote catalog or warehouse identifier is required"); + } + + @Test + void rejectsMissingQuotaProjectHeader() { + assertThatThrownBy( + () -> + BigLakeCatalogValidator.validate( + realmConfig, + bigLakeCatalog( + "https://biglake.googleapis.com/iceberg/v1/restcatalog", + "my-remote-catalog", + Map.of(), + true, + "gs://bucket/path/to/data", + validGcsStorage("gs://bucket/path/to/data")))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("a quota project is required"); + } + + @Test + void rejectsUnsupportedHeaderOverride() { + assertThatThrownBy( + () -> + BigLakeCatalogValidator.validate( + realmConfig, + bigLakeCatalog( + "https://biglake.googleapis.com/iceberg/v1/restcatalog", + "my-remote-catalog", + Map.of( + "header.x-goog-user-project", + "my-billing-project", + "header.authorization", + "Bearer secret"), + true, + "gs://bucket/path/to/data", + validGcsStorage("gs://bucket/path/to/data")))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("overriding security-sensitive headers is not allowed"); + } + + @Test + void rejectsMissingGcsStorageWhenCredentialVendingEnabled() { + assertThatThrownBy( + () -> + BigLakeCatalogValidator.validate( + realmConfig, + bigLakeCatalog( + "https://biglake.googleapis.com/iceberg/v1/restcatalog", + "my-remote-catalog", + Map.of("header.x-goog-user-project", "my-billing-project"), + true, + "gs://bucket/path/to/data", + null))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("GCS storage configuration is required"); + } + + @Test + void rejectsMalformedGsBaseLocation() { + assertThatThrownBy( + () -> + BigLakeCatalogValidator.validate( + realmConfig, + bigLakeCatalog( + "https://biglake.googleapis.com/iceberg/v1/restcatalog", + "my-remote-catalog", + Map.of("header.x-goog-user-project", "my-billing-project"), + true, + "s3://bucket/path/to/data", + validGcsStorage("gs://bucket/path/to/data")))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("catalog.properties.default-base-location"); + } + + @Test + void rejectsMalformedGsAllowedLocation() { + assertThatThrownBy( + () -> + BigLakeCatalogValidator.validate( + realmConfig, + bigLakeCatalog( + "https://biglake.googleapis.com/iceberg/v1/restcatalog", + "my-remote-catalog", + Map.of("header.x-goog-user-project", "my-billing-project"), + true, + "gs://bucket/path/to/data", + invalidAllowedLocationStorage()))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("storageConfigInfo.allowedLocations[0]"); + } + + @Test + void rejectsInvalidGcsServiceAccount() { + assertThatThrownBy( + () -> + BigLakeCatalogValidator.validate( + realmConfig, + bigLakeCatalog( + "https://biglake.googleapis.com/iceberg/v1/restcatalog", + "my-remote-catalog", + Map.of("header.x-goog-user-project", "my-billing-project"), + true, + "gs://bucket/path/to/data", + GcpStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.GCS) + .setGcsServiceAccount("not-a-service-account") + .setAllowedLocations(List.of("gs://bucket/path/to/data")) + .build()))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("expected a Google service account email"); + } + + private Catalog bigLakeCatalog( + String uri, + String remoteCatalogName, + Map connectionProperties, + boolean credentialVendingEnabled, + String defaultBaseLocation, + StorageConfigInfo storageConfigInfo) { + CatalogProperties catalogProperties = CatalogProperties.builder(defaultBaseLocation).build(); + catalogProperties.put( + "enable.credential.vending", Boolean.toString(credentialVendingEnabled)); + + return ExternalCatalog.builder() + .setType(Catalog.TypeEnum.EXTERNAL) + .setName("test-biglake-catalog") + .setProperties(catalogProperties) + .setStorageConfigInfo(storageConfigInfo) + .setConnectionConfigInfo( + IcebergRestConnectionConfigInfo.builder() + .setConnectionType(ConnectionConfigInfo.ConnectionTypeEnum.ICEBERG_REST) + .setUri(uri) + .setRemoteCatalogName(remoteCatalogName) + .setProperties(connectionProperties) + .setAuthenticationParameters( + GcpAuthenticationParameters.builder() + .setAuthenticationType(AuthenticationParameters.AuthenticationTypeEnum.GCP) + .build()) + .build()) + .build(); + } + + private StorageConfigInfo validGcsStorage(String allowedLocation) { + return GcpStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.GCS) + .setGcsServiceAccount("test-sa@my-project.iam.gserviceaccount.com") + .setAllowedLocations(List.of(allowedLocation)) + .build(); + } + + private StorageConfigInfo invalidAllowedLocationStorage() { + return GcpStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.GCS) + .setGcsServiceAccount("test-sa@my-project.iam.gserviceaccount.com") + .setAllowedLocations(List.of("bucket/path/to/data")) + .build(); + } +} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java b/runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java index 52ab0a5cefe..f6974ae57e8 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java @@ -38,6 +38,8 @@ import org.apache.polaris.core.admin.model.CreateCatalogRequest; import org.apache.polaris.core.admin.model.ExternalCatalog; import org.apache.polaris.core.admin.model.FileStorageConfigInfo; +import org.apache.polaris.core.admin.model.GcpAuthenticationParameters; +import org.apache.polaris.core.admin.model.GcpStorageConfigInfo; import org.apache.polaris.core.admin.model.IcebergRestConnectionConfigInfo; import org.apache.polaris.core.admin.model.OAuthClientCredentialsParameters; import org.apache.polaris.core.admin.model.PolarisCatalog; @@ -369,6 +371,117 @@ public void testUpdateCatalogWithDisallowedConfigs() { "Explicitly setting polaris.config.enable-sub-catalog-rbac-for-federated-catalogs is not allowed because ALLOW_SETTING_SUB_CATALOG_RBAC_FOR_FEDERATED_CATALOGS is set to false."); } + + @Test + public void testCreateAndUpdateValidBigLakeCatalog() { + String catalogName = "biglake-catalog"; + String initialBaseLocation = "gs://bucket/path/to/data"; + String updatedBaseLocation = "gs://bucket/path/to/updated-data"; + Catalog catalog = + createBigLakeCatalog( + catalogName, + initialBaseLocation, + createBigLakeStorageConfig(initialBaseLocation)); + + try (Response response = + services + .catalogsApi() + .createCatalog( + new CreateCatalogRequest(catalog), + services.realmContext(), + services.securityContext())) { + assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); + } + + Catalog fetchedCatalog; + try (Response response = + services + .catalogsApi() + .getCatalog(catalogName, services.realmContext(), services.securityContext())) { + assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); + fetchedCatalog = (Catalog) response.getEntity(); + } + + UpdateCatalogRequest updateRequest = + UpdateCatalogRequest.builder() + .setCurrentEntityVersion(fetchedCatalog.getEntityVersion()) + .setProperties( + Map.of( + "default-base-location", + updatedBaseLocation, + "enable.credential.vending", + "true")) + .setStorageConfigInfo(createBigLakeStorageConfig(updatedBaseLocation)) + .build(); + + try (Response response = + services + .catalogsApi() + .updateCatalog( + catalogName, + updateRequest, + services.realmContext(), + services.securityContext())) { + assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); + Catalog updatedCatalog = (Catalog) response.getEntity(); + assertThat(updatedCatalog.getProperties().getDefaultBaseLocation()) + .isEqualTo(updatedBaseLocation); + } + } + + @Test + public void testUpdateBigLakeCatalogRejectsInvalidMergedConfiguration() { + String catalogName = "biglake-catalog"; + String initialBaseLocation = "gs://bucket/path/to/data"; + Catalog catalog = + createBigLakeCatalog( + catalogName, + initialBaseLocation, + createBigLakeStorageConfig(initialBaseLocation)); + + try (Response response = + services + .catalogsApi() + .createCatalog( + new CreateCatalogRequest(catalog), + services.realmContext(), + services.securityContext())) { + assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); + } + + Catalog fetchedCatalog; + try (Response response = + services + .catalogsApi() + .getCatalog(catalogName, services.realmContext(), services.securityContext())) { + assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); + fetchedCatalog = (Catalog) response.getEntity(); + } + + UpdateCatalogRequest updateRequest = + UpdateCatalogRequest.builder() + .setCurrentEntityVersion(fetchedCatalog.getEntityVersion()) + .setProperties( + Map.of( + "default-base-location", + "s3://bucket/path/to/data", + "enable.credential.vending", + "true")) + .build(); + + assertThatThrownBy( + () -> + services + .catalogsApi() + .updateCatalog( + catalogName, + updateRequest, + services.realmContext(), + services.securityContext())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("catalog.properties.default-base-location"); + } + private PolarisAdminService setupPolarisAdminService( PolarisMetaStoreManager metaStoreManager, PolarisCallContext callContext) { PrincipalEntity rootPrincipal = @@ -391,6 +504,38 @@ private PolarisAdminService setupPolarisAdminService( ReservedProperties.NONE); } + + private Catalog createBigLakeCatalog( + String catalogName, String defaultBaseLocation, StorageConfigInfo storageConfigInfo) { + CatalogProperties catalogProperties = CatalogProperties.builder(defaultBaseLocation).build(); + catalogProperties.put("enable.credential.vending", "true"); + return ExternalCatalog.builder() + .setType(Catalog.TypeEnum.EXTERNAL) + .setName(catalogName) + .setProperties(catalogProperties) + .setStorageConfigInfo(storageConfigInfo) + .setConnectionConfigInfo( + IcebergRestConnectionConfigInfo.builder( + ConnectionConfigInfo.ConnectionTypeEnum.ICEBERG_REST) + .setUri("https://biglake.googleapis.com/iceberg/v1/restcatalog") + .setRemoteCatalogName("my-remote-catalog") + .setProperties(Map.of("header.x-goog-user-project", "my-billing-project")) + .setAuthenticationParameters( + GcpAuthenticationParameters.builder() + .setAuthenticationType(AuthenticationParameters.AuthenticationTypeEnum.GCP) + .build()) + .build()) + .build(); + } + + private StorageConfigInfo createBigLakeStorageConfig(String allowedLocation) { + return GcpStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.GCS) + .setGcsServiceAccount("test-sa@my-project.iam.gserviceaccount.com") + .setAllowedLocations(List.of(allowedLocation)) + .build(); + } + private PrincipalEntity createPrincipal( PolarisMetaStoreManager metaStoreManager, PolarisCallContext callContext, String name) { return new PrincipalEntity.Builder() From 7d61f9a4a4d45328926a0f8fb96961693a88a3be Mon Sep 17 00:00:00 2001 From: Garik Kalashyan Date: Wed, 5 Aug 2026 13:33:32 +0400 Subject: [PATCH 2/6] BigLake validation PR improves fail-fast behavior for BigLake external catalogs --- .../admin/BigLakeCatalogValidator.java | 51 ++++++--- .../admin/BigLakeCatalogValidatorTest.java | 101 +++++++++++++++++- .../service/admin/ManagementServiceTest.java | 87 +++++++++++++++ 3 files changed, 226 insertions(+), 13 deletions(-) diff --git a/runtime/service/src/main/java/org/apache/polaris/service/admin/BigLakeCatalogValidator.java b/runtime/service/src/main/java/org/apache/polaris/service/admin/BigLakeCatalogValidator.java index 0262f3d88ad..3434bef664c 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/admin/BigLakeCatalogValidator.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/admin/BigLakeCatalogValidator.java @@ -44,6 +44,8 @@ final class BigLakeCatalogValidator { private static final Pattern GCP_PROJECT_ID_PATTERN = Pattern.compile("^[a-z][a-z0-9-]{4,28}[a-z0-9]$"); private static final Pattern GCP_PROJECT_NUMBER_PATTERN = Pattern.compile("^[1-9][0-9]{5,}$"); + private static final Pattern BIGLAKE_URI_CATALOG_PATTERN = + Pattern.compile("^/[1-9][0-9]{5,}/catalogs/[^/\\s]+$"); private static final Pattern BIGLAKE_RESOURCE_NAME_PATTERN = Pattern.compile("^projects/[^/\\s]+/locations/[^/\\s]+/catalogs/[^/\\s]+$"); private static final Pattern BIGLAKE_SIMPLE_CATALOG_PATTERN = @@ -71,26 +73,23 @@ static void validate(RealmConfig realmConfig, Catalog catalog) { return; } - validateBigLakeEndpoint(connectionConfig.getUri()); + URI uri = parseUri(connectionConfig.getUri()); + if (!targetsBigLakeHost(uri)) { + return; + } + + validateBigLakeEndpoint(connectionConfig.getUri(), uri); validateBigLakeRemoteCatalogName(connectionConfig.getRemoteCatalogName()); validateBigLakeHeaders(connectionConfig.getProperties()); validateBigLakeStorageConfiguration(realmConfig, externalCatalog); } - private static void validateBigLakeEndpoint(String uriString) { + private static void validateBigLakeEndpoint(String uriString, URI uri) { if (Strings.isNullOrEmpty(uriString)) { throw new IllegalArgumentException( "Invalid BigLake connectionConfigInfo.uri: an https:// BigLake endpoint is required."); } - URI uri; - try { - uri = URI.create(uriString); - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException( - "Invalid BigLake connectionConfigInfo.uri '" + uriString + "'.", e); - } - if (!"https".equalsIgnoreCase(uri.getScheme())) { throw new IllegalArgumentException( "Invalid BigLake connectionConfigInfo.uri '" @@ -141,7 +140,8 @@ private static void validateBigLakeRemoteCatalogName(String remoteCatalogName) { return; } - if (BIGLAKE_RESOURCE_NAME_PATTERN.matcher(trimmedRemoteCatalogName).matches() + if (isBigLakeCatalogUri(trimmedRemoteCatalogName) + || BIGLAKE_RESOURCE_NAME_PATTERN.matcher(trimmedRemoteCatalogName).matches() || BIGLAKE_SIMPLE_CATALOG_PATTERN.matcher(trimmedRemoteCatalogName).matches()) { return; } @@ -272,10 +272,37 @@ private static void validateGsLocation(String fieldName, String location) { } } + private static URI parseUri(String uriString) { + if (Strings.isNullOrEmpty(uriString)) { + return null; + } + + try { + return URI.create(uriString); + } catch (IllegalArgumentException e) { + return null; + } + } + + private static boolean targetsBigLakeHost(URI uri) { + return uri != null && BIGLAKE_HOST.equalsIgnoreCase(uri.getHost()); + } + + private static boolean isBigLakeCatalogUri(String remoteCatalogName) { + URI uri = parseUri(remoteCatalogName); + return uri != null + && "bl".equalsIgnoreCase(uri.getScheme()) + && "projects".equalsIgnoreCase(uri.getHost()) + && uri.getPort() == -1 + && uri.getRawQuery() == null + && uri.getRawFragment() == null + && BIGLAKE_URI_CATALOG_PATTERN.matcher(normalizePath(uri.getPath())).matches(); + } + private static String normalizePath(String path) { if (Strings.isNullOrEmpty(path)) { return ""; } return path.endsWith("/") && path.length() > 1 ? path.substring(0, path.length() - 1) : path; } -} +} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/admin/BigLakeCatalogValidatorTest.java b/runtime/service/src/test/java/org/apache/polaris/service/admin/BigLakeCatalogValidatorTest.java index 58d8d49d3c8..6c65581bc70 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/admin/BigLakeCatalogValidatorTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/admin/BigLakeCatalogValidatorTest.java @@ -61,6 +61,54 @@ void validBigLakeConfigurationPasses() { .doesNotThrowAnyException(); } + @Test + void validBigLakeBlCatalogIdentifierPasses() { + assertThatCode( + () -> + BigLakeCatalogValidator.validate( + realmConfig, + bigLakeCatalog( + "https://biglake.googleapis.com/iceberg/v1/restcatalog", + "bl://projects/123456789/catalogs/my-biglake-catalog", + Map.of("header.x-goog-user-project", "my-billing-project"), + true, + "gs://bucket/path/to/data", + validGcsStorage("gs://bucket/path/to/data")))) + .doesNotThrowAnyException(); + } + + @Test + void validBigLakeWarehouseIdentifierPasses() { + assertThatCode( + () -> + BigLakeCatalogValidator.validate( + realmConfig, + bigLakeCatalog( + "https://biglake.googleapis.com/iceberg/v1/restcatalog", + "gs://bucket/path/to/warehouse", + Map.of("header.x-goog-user-project", "my-billing-project"), + true, + "gs://bucket/path/to/data", + validGcsStorage("gs://bucket/path/to/data")))) + .doesNotThrowAnyException(); + } + + @Test + void skipsValidationForNonBigLakeGcpRestEndpoint() { + assertThatCode( + () -> + BigLakeCatalogValidator.validate( + realmConfig, + bigLakeCatalog( + "https://catalog-gateway.example.com/iceberg/v1", + null, + Map.of(), + true, + "s3://bucket/path/to/data", + null))) + .doesNotThrowAnyException(); + } + @Test void rejectsNonHttpsEndpoint() { assertThatThrownBy( @@ -95,6 +143,57 @@ void rejectsUnsupportedEndpointPath() { .hasMessageContaining("unsupported path"); } + @Test + void rejectsEndpointWithQueryString() { + assertThatThrownBy( + () -> + BigLakeCatalogValidator.validate( + realmConfig, + bigLakeCatalog( + "https://biglake.googleapis.com/iceberg/v1/restcatalog?warehouse=test", + "my-remote-catalog", + Map.of("header.x-goog-user-project", "my-billing-project"), + true, + "gs://bucket/path/to/data", + validGcsStorage("gs://bucket/path/to/data")))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("query, fragment, and custom port components are not supported"); + } + + @Test + void rejectsEndpointWithFragment() { + assertThatThrownBy( + () -> + BigLakeCatalogValidator.validate( + realmConfig, + bigLakeCatalog( + "https://biglake.googleapis.com/iceberg/v1/restcatalog#fragment", + "my-remote-catalog", + Map.of("header.x-goog-user-project", "my-billing-project"), + true, + "gs://bucket/path/to/data", + validGcsStorage("gs://bucket/path/to/data")))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("query, fragment, and custom port components are not supported"); + } + + @Test + void rejectsEndpointWithCustomPort() { + assertThatThrownBy( + () -> + BigLakeCatalogValidator.validate( + realmConfig, + bigLakeCatalog( + "https://biglake.googleapis.com:8443/iceberg/v1/restcatalog", + "my-remote-catalog", + Map.of("header.x-goog-user-project", "my-billing-project"), + true, + "gs://bucket/path/to/data", + validGcsStorage("gs://bucket/path/to/data")))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("query, fragment, and custom port components are not supported"); + } + @Test void rejectsMissingRemoteCatalogIdentifier() { assertThatThrownBy( @@ -267,4 +366,4 @@ private StorageConfigInfo invalidAllowedLocationStorage() { .setAllowedLocations(List.of("bucket/path/to/data")) .build(); } -} +} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java b/runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java index f6974ae57e8..871a688b036 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java @@ -429,6 +429,60 @@ public void testCreateAndUpdateValidBigLakeCatalog() { } } + @Test + public void testCreateAndUpdateNonBigLakeGcpRestCatalogSkipsBigLakeValidation() { + String catalogName = "generic-gcp-rest-catalog"; + String initialBaseLocation = "s3://bucket/path/to/data"; + String updatedBaseLocation = "s3://bucket/path/to/updated-data"; + Catalog catalog = + createGenericGcpRestCatalog( + catalogName, initialBaseLocation, createGenericS3StorageConfig(initialBaseLocation)); + + try (Response response = + services + .catalogsApi() + .createCatalog( + new CreateCatalogRequest(catalog), + services.realmContext(), + services.securityContext())) { + assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); + } + + Catalog fetchedCatalog; + try (Response response = + services + .catalogsApi() + .getCatalog(catalogName, services.realmContext(), services.securityContext())) { + assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); + fetchedCatalog = (Catalog) response.getEntity(); + assertThat(fetchedCatalog.getProperties().getDefaultBaseLocation()) + .isEqualTo(initialBaseLocation); + } + + UpdateCatalogRequest updateRequest = + UpdateCatalogRequest.builder() + .setCurrentEntityVersion(fetchedCatalog.getEntityVersion()) + .setProperties( + Map.of( + "default-base-location", + updatedBaseLocation, + "enable.credential.vending", + "true")) + .setStorageConfigInfo(createGenericS3StorageConfig(updatedBaseLocation)) + .build(); + + try (Response response = + services + .catalogsApi() + .updateCatalog( + catalogName, updateRequest, services.realmContext(), services.securityContext())) { + assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); + Catalog updatedCatalog = (Catalog) response.getEntity(); + assertThat(updatedCatalog.getProperties().getDefaultBaseLocation()) + .isEqualTo(updatedBaseLocation); + } + } + @Test public void testUpdateBigLakeCatalogRejectsInvalidMergedConfiguration() { String catalogName = "biglake-catalog"; @@ -528,6 +582,28 @@ private Catalog createBigLakeCatalog( .build(); } + private Catalog createGenericGcpRestCatalog( + String catalogName, String defaultBaseLocation, StorageConfigInfo storageConfigInfo) { + CatalogProperties catalogProperties = CatalogProperties.builder(defaultBaseLocation).build(); + catalogProperties.put("enable.credential.vending", "true"); + return ExternalCatalog.builder() + .setType(Catalog.TypeEnum.EXTERNAL) + .setName(catalogName) + .setProperties(catalogProperties) + .setStorageConfigInfo(storageConfigInfo) + .setConnectionConfigInfo( + IcebergRestConnectionConfigInfo.builder( + ConnectionConfigInfo.ConnectionTypeEnum.ICEBERG_REST) + .setUri("https://catalog-gateway.example.com/iceberg/v1") + .setRemoteCatalogName("my-remote-catalog") + .setAuthenticationParameters( + GcpAuthenticationParameters.builder() + .setAuthenticationType(AuthenticationParameters.AuthenticationTypeEnum.GCP) + .build()) + .build()) + .build(); + } + private StorageConfigInfo createBigLakeStorageConfig(String allowedLocation) { return GcpStorageConfigInfo.builder() .setStorageType(StorageConfigInfo.StorageTypeEnum.GCS) @@ -536,6 +612,16 @@ private StorageConfigInfo createBigLakeStorageConfig(String allowedLocation) { .build(); } + private StorageConfigInfo createGenericS3StorageConfig(String allowedLocation) { + return AwsStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) + .setRoleArn("arn:aws:iam::123456789012:role/my-role") + .setExternalId("externalId") + .setUserArn("userArn") + .setAllowedLocations(List.of(allowedLocation)) + .build(); + } + private PrincipalEntity createPrincipal( PolarisMetaStoreManager metaStoreManager, PolarisCallContext callContext, String name) { return new PrincipalEntity.Builder() @@ -895,3 +981,4 @@ public void testCreateCatalogReturnErrorOnFailure() { resultWithError.getExtraInformation())); } } + From 25bf3cf5ac4752be57f437c840b37caa41b4c463 Mon Sep 17 00:00:00 2001 From: "david.chaava" Date: Wed, 5 Aug 2026 12:20:02 +0200 Subject: [PATCH 3/6] Fix BigLake CLI quota project configuration --- .../apache_polaris/cli/command/catalogs.py | 17 +++++++++- client/python/tests/test_catalogs_command.py | 4 +++ .../admin/BigLakeCatalogValidator.java | 18 ++++++++--- .../admin/BigLakeCatalogValidatorTest.java | 31 +++++++++++++++++++ 4 files changed, 64 insertions(+), 6 deletions(-) diff --git a/client/python/apache_polaris/cli/command/catalogs.py b/client/python/apache_polaris/cli/command/catalogs.py index fae73d75261..9dee6666b03 100644 --- a/client/python/apache_polaris/cli/command/catalogs.py +++ b/client/python/apache_polaris/cli/command/catalogs.py @@ -74,6 +74,8 @@ class CatalogsCommand(Command): * polaris catalogs list """ + _GCP_QUOTA_PROJECT_PROPERTY = "header.x-goog-user-project" + catalogs_subcommand: str catalog_type: Optional[str] = None default_base_location: Optional[str] = None @@ -401,12 +403,19 @@ def _build_connection_config_info( warehouse=self.hadoop_warehouse, ) elif self.catalog_connection_type == CatalogConnectionType.ICEBERG.value: + connection_properties = {} + if self.catalog_authentication_type == AuthenticationType.GCP.value: + quota_project = self.properties.get(self._GCP_QUOTA_PROJECT_PROPERTY) + if quota_project is not None: + connection_properties[self._GCP_QUOTA_PROJECT_PROPERTY] = quota_project + config = IcebergRestConnectionConfigInfo( connection_type=self.catalog_connection_type.upper().replace("-", "_"), uri=self.catalog_uri, authentication_parameters=auth_params, service_identity=service_identity, remote_catalog_name=self.iceberg_remote_catalog_name, + properties=connection_properties, ) elif self.catalog_connection_type == CatalogConnectionType.HIVE.value: config = HiveConnectionConfigInfo( @@ -425,6 +434,12 @@ def _build_connection_config_info( def execute(self, api: PolarisDefaultApi) -> None: catalog_type = cast(str, self.catalog_type) catalog_name = cast(str, self.catalog_name) + catalog_properties = dict(self.properties) + if ( + self.catalog_connection_type == CatalogConnectionType.ICEBERG.value + and self.catalog_authentication_type == AuthenticationType.GCP.value + ): + catalog_properties.pop(self._GCP_QUOTA_PROJECT_PROPERTY, None) if self.catalogs_subcommand == Subcommands.CREATE: storage_config = self._build_storage_config_info() @@ -437,7 +452,7 @@ def execute(self, api: PolarisDefaultApi) -> None: storage_config_info=storage_config, properties=CatalogProperties( default_base_location=self.default_base_location, - additional_properties=self.properties, + additional_properties=catalog_properties, ), connection_config_info=connection_config, ) diff --git a/client/python/tests/test_catalogs_command.py b/client/python/tests/test_catalogs_command.py index 3a159dfc080..7a8ce648e9a 100644 --- a/client/python/tests/test_catalogs_command.py +++ b/client/python/tests/test_catalogs_command.py @@ -747,6 +747,10 @@ def test_external_catalog_gcp(self) -> None: self.assertEqual(call_args.catalog.properties.default_base_location, "dbl") self.assertEqual( call_args.catalog.properties.additional_properties, + {}, + ) + self.assertEqual( + call_args.catalog.connection_config_info.properties, {"header.x-goog-user-project": "my-billing-project"}, ) diff --git a/runtime/service/src/main/java/org/apache/polaris/service/admin/BigLakeCatalogValidator.java b/runtime/service/src/main/java/org/apache/polaris/service/admin/BigLakeCatalogValidator.java index 3434bef664c..b968216eef6 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/admin/BigLakeCatalogValidator.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/admin/BigLakeCatalogValidator.java @@ -80,7 +80,8 @@ static void validate(RealmConfig realmConfig, Catalog catalog) { validateBigLakeEndpoint(connectionConfig.getUri(), uri); validateBigLakeRemoteCatalogName(connectionConfig.getRemoteCatalogName()); - validateBigLakeHeaders(connectionConfig.getProperties()); + validateBigLakeHeaders( + connectionConfig.getProperties(), externalCatalog.getProperties().toMap()); validateBigLakeStorageConfiguration(realmConfig, externalCatalog); } @@ -152,8 +153,10 @@ private static void validateBigLakeRemoteCatalogName(String remoteCatalogName) { + "': expected a BigLake catalog identifier or gs:// warehouse location."); } - private static void validateBigLakeHeaders(Map properties) { - Map headerProperties = properties != null ? properties : Map.of(); + private static void validateBigLakeHeaders( + Map connectionProperties, Map catalogProperties) { + Map headerProperties = + connectionProperties != null ? connectionProperties : Map.of(); for (String propertyName : headerProperties.keySet()) { if (propertyName == null) { @@ -182,9 +185,14 @@ private static void validateBigLakeHeaders(Map properties) { } String quotaProject = headerProperties.get(QUOTA_PROJECT_HEADER); + if (Strings.isNullOrEmpty(quotaProject) && catalogProperties != null) { + // Preserve existing CLI-created catalogs while new CLI requests store this header on the + // connection configuration, where it is used for outbound BigLake requests. + quotaProject = catalogProperties.get(QUOTA_PROJECT_HEADER); + } if (Strings.isNullOrEmpty(quotaProject) || quotaProject.trim().isEmpty()) { throw new IllegalArgumentException( - "Invalid BigLake connectionConfigInfo.properties entry '" + "Invalid BigLake connectionConfigInfo.properties entry or catalog.properties entry '" + QUOTA_PROJECT_HEADER + "': a quota project is required."); } @@ -193,7 +201,7 @@ private static void validateBigLakeHeaders(Map properties) { if (!GCP_PROJECT_ID_PATTERN.matcher(trimmedQuotaProject).matches() && !GCP_PROJECT_NUMBER_PATTERN.matcher(trimmedQuotaProject).matches()) { throw new IllegalArgumentException( - "Invalid BigLake connectionConfigInfo.properties entry '" + "Invalid BigLake connectionConfigInfo.properties entry or catalog.properties entry '" + QUOTA_PROJECT_HEADER + "': '" + quotaProject diff --git a/runtime/service/src/test/java/org/apache/polaris/service/admin/BigLakeCatalogValidatorTest.java b/runtime/service/src/test/java/org/apache/polaris/service/admin/BigLakeCatalogValidatorTest.java index 6c65581bc70..903d77a610b 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/admin/BigLakeCatalogValidatorTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/admin/BigLakeCatalogValidatorTest.java @@ -61,6 +61,37 @@ void validBigLakeConfigurationPasses() { .doesNotThrowAnyException(); } + @Test + void validBigLakeConfigurationWithLegacyCatalogQuotaProjectPasses() { + CatalogProperties catalogProperties = + CatalogProperties.builder("gs://bucket/path/to/data").build(); + catalogProperties.put("enable.credential.vending", "true"); + catalogProperties.put("header.x-goog-user-project", "my-billing-project"); + + Catalog catalog = + ExternalCatalog.builder() + .setType(Catalog.TypeEnum.EXTERNAL) + .setName("test-biglake-catalog") + .setProperties(catalogProperties) + .setStorageConfigInfo(validGcsStorage("gs://bucket/path/to/data")) + .setConnectionConfigInfo( + IcebergRestConnectionConfigInfo.builder() + .setConnectionType(ConnectionConfigInfo.ConnectionTypeEnum.ICEBERG_REST) + .setUri("https://biglake.googleapis.com/iceberg/v1/restcatalog") + .setRemoteCatalogName("my-remote-catalog") + .setProperties(Map.of()) + .setAuthenticationParameters( + GcpAuthenticationParameters.builder() + .setAuthenticationType( + AuthenticationParameters.AuthenticationTypeEnum.GCP) + .build()) + .build()) + .build(); + + assertThatCode(() -> BigLakeCatalogValidator.validate(realmConfig, catalog)) + .doesNotThrowAnyException(); + } + @Test void validBigLakeBlCatalogIdentifierPasses() { assertThatCode( From 79835cba0e5f5fb5ccb705e169a8f5b64ec895e4 Mon Sep 17 00:00:00 2001 From: "david.chaava" Date: Wed, 5 Aug 2026 17:15:23 +0200 Subject: [PATCH 4/6] Fix BigLake validation CI checks --- client/python/apache_polaris/cli/command/catalogs.py | 10 +++++++--- .../polaris/service/admin/BigLakeCatalogValidator.java | 2 +- .../service/admin/BigLakeCatalogValidatorTest.java | 2 +- .../polaris/service/admin/ManagementServiceTest.java | 1 - 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/client/python/apache_polaris/cli/command/catalogs.py b/client/python/apache_polaris/cli/command/catalogs.py index 9dee6666b03..9c240b82577 100644 --- a/client/python/apache_polaris/cli/command/catalogs.py +++ b/client/python/apache_polaris/cli/command/catalogs.py @@ -405,9 +405,13 @@ def _build_connection_config_info( elif self.catalog_connection_type == CatalogConnectionType.ICEBERG.value: connection_properties = {} if self.catalog_authentication_type == AuthenticationType.GCP.value: - quota_project = self.properties.get(self._GCP_QUOTA_PROJECT_PROPERTY) + quota_project = (self.properties or {}).get( + self._GCP_QUOTA_PROJECT_PROPERTY + ) if quota_project is not None: - connection_properties[self._GCP_QUOTA_PROJECT_PROPERTY] = quota_project + connection_properties[self._GCP_QUOTA_PROJECT_PROPERTY] = ( + quota_project + ) config = IcebergRestConnectionConfigInfo( connection_type=self.catalog_connection_type.upper().replace("-", "_"), @@ -434,7 +438,7 @@ def _build_connection_config_info( def execute(self, api: PolarisDefaultApi) -> None: catalog_type = cast(str, self.catalog_type) catalog_name = cast(str, self.catalog_name) - catalog_properties = dict(self.properties) + catalog_properties = dict(self.properties or {}) if ( self.catalog_connection_type == CatalogConnectionType.ICEBERG.value and self.catalog_authentication_type == AuthenticationType.GCP.value diff --git a/runtime/service/src/main/java/org/apache/polaris/service/admin/BigLakeCatalogValidator.java b/runtime/service/src/main/java/org/apache/polaris/service/admin/BigLakeCatalogValidator.java index b968216eef6..d3dde4596a3 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/admin/BigLakeCatalogValidator.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/admin/BigLakeCatalogValidator.java @@ -313,4 +313,4 @@ private static String normalizePath(String path) { } return path.endsWith("/") && path.length() > 1 ? path.substring(0, path.length() - 1) : path; } -} +} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/admin/BigLakeCatalogValidatorTest.java b/runtime/service/src/test/java/org/apache/polaris/service/admin/BigLakeCatalogValidatorTest.java index 903d77a610b..cccc7f96f1e 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/admin/BigLakeCatalogValidatorTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/admin/BigLakeCatalogValidatorTest.java @@ -397,4 +397,4 @@ private StorageConfigInfo invalidAllowedLocationStorage() { .setAllowedLocations(List.of("bucket/path/to/data")) .build(); } -} +} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java b/runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java index 871a688b036..5877eebabd1 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java @@ -981,4 +981,3 @@ public void testCreateCatalogReturnErrorOnFailure() { resultWithError.getExtraInformation())); } } - From 91273940074365690dbaa7a9fbd53202b86a6425 Mon Sep 17 00:00:00 2001 From: David Chaava Date: Fri, 14 Aug 2026 12:17:51 +0200 Subject: [PATCH 5/6] Relax BigLake service account email validation --- .../admin/BigLakeCatalogValidator.java | 9 ++--- .../admin/BigLakeCatalogValidatorTest.java | 33 ++++++++++++++++++- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/runtime/service/src/main/java/org/apache/polaris/service/admin/BigLakeCatalogValidator.java b/runtime/service/src/main/java/org/apache/polaris/service/admin/BigLakeCatalogValidator.java index d3dde4596a3..fd5aaddbd31 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/admin/BigLakeCatalogValidator.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/admin/BigLakeCatalogValidator.java @@ -50,8 +50,8 @@ final class BigLakeCatalogValidator { Pattern.compile("^projects/[^/\\s]+/locations/[^/\\s]+/catalogs/[^/\\s]+$"); private static final Pattern BIGLAKE_SIMPLE_CATALOG_PATTERN = Pattern.compile("^[A-Za-z0-9._-]+$"); - private static final Pattern GCS_SERVICE_ACCOUNT_PATTERN = - Pattern.compile("^[a-z][a-z0-9-]{4,28}[a-z0-9]@[a-z][a-z0-9-]{4,28}[a-z0-9]\\.iam\\.gserviceaccount\\.com$"); + private static final Pattern SERVICE_ACCOUNT_EMAIL_PATTERN = + Pattern.compile("^[^\\s@]+@[^\\s@]+\\.[^\\s@]+$"); private static final Set BLOCKED_HEADER_PROPERTIES = Set.of("header.authorization", "header.proxy-authorization"); @@ -247,11 +247,12 @@ private static void validateBigLakeStorageConfiguration( } String serviceAccount = gcpStorageConfigInfo.getGcsServiceAccount(); - if (!Strings.isNullOrEmpty(serviceAccount) && !GCS_SERVICE_ACCOUNT_PATTERN.matcher(serviceAccount).matches()) { + if (!Strings.isNullOrEmpty(serviceAccount) + && !SERVICE_ACCOUNT_EMAIL_PATTERN.matcher(serviceAccount).matches()) { throw new IllegalArgumentException( "Invalid BigLake storageConfigInfo.gcsServiceAccount '" + serviceAccount - + "': expected a Google service account email."); + + "': expected a syntactically valid service account email."); } if (credentialVendingEnabled && Strings.isNullOrEmpty(serviceAccount)) { diff --git a/runtime/service/src/test/java/org/apache/polaris/service/admin/BigLakeCatalogValidatorTest.java b/runtime/service/src/test/java/org/apache/polaris/service/admin/BigLakeCatalogValidatorTest.java index cccc7f96f1e..322ea426393 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/admin/BigLakeCatalogValidatorTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/admin/BigLakeCatalogValidatorTest.java @@ -36,6 +36,8 @@ import org.apache.polaris.service.TestServices; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; class BigLakeCatalogValidatorTest { private RealmConfig realmConfig; @@ -349,7 +351,36 @@ void rejectsInvalidGcsServiceAccount() { .setAllowedLocations(List.of("gs://bucket/path/to/data")) .build()))) .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("expected a Google service account email"); + .hasMessageContaining("expected a syntactically valid service account email"); + } + + @ParameterizedTest + @ValueSource( + strings = { + "test-sa@my-project.iam.gserviceaccount.com", + "123456789-compute@developer.gserviceaccount.com", + "my-project@appspot.gserviceaccount.com" + }) + void acceptsSupportedGcsServiceAccountEmailForms(String serviceAccount) { + StorageConfigInfo storageConfigInfo = + GcpStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.GCS) + .setGcsServiceAccount(serviceAccount) + .setAllowedLocations(List.of("gs://bucket/path/to/data")) + .build(); + + assertThatCode( + () -> + BigLakeCatalogValidator.validate( + realmConfig, + bigLakeCatalog( + "https://biglake.googleapis.com/iceberg/v1/restcatalog", + "my-remote-catalog", + Map.of("header.x-goog-user-project", "my-billing-project"), + true, + "gs://bucket/path/to/data", + storageConfigInfo))) + .doesNotThrowAnyException(); } private Catalog bigLakeCatalog( From bf5d04d0bae123a68fd0c4ff49a7610b024e2474 Mon Sep 17 00:00:00 2001 From: Garik Kalashyan Date: Mon, 17 Aug 2026 13:50:50 +0400 Subject: [PATCH 6/6] spotless apply + test fix --- .../service/admin/BigLakeCatalogValidator.java | 12 ++++++++---- .../admin/BigLakeCatalogValidatorTest.java | 3 +-- .../service/admin/ManagementServiceTest.java | 17 ++++------------- 3 files changed, 13 insertions(+), 19 deletions(-) diff --git a/runtime/service/src/main/java/org/apache/polaris/service/admin/BigLakeCatalogValidator.java b/runtime/service/src/main/java/org/apache/polaris/service/admin/BigLakeCatalogValidator.java index fd5aaddbd31..13e175f40ea 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/admin/BigLakeCatalogValidator.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/admin/BigLakeCatalogValidator.java @@ -63,7 +63,8 @@ static void validate(RealmConfig realmConfig, Catalog catalog) { return; } - if (!(externalCatalog.getConnectionConfigInfo() instanceof IcebergRestConnectionConfigInfo connectionConfig)) { + if (!(externalCatalog.getConnectionConfigInfo() + instanceof IcebergRestConnectionConfigInfo connectionConfig)) { return; } @@ -236,13 +237,15 @@ private static void validateBigLakeStorageConfiguration( + "."); } - String defaultBaseLocation = externalCatalog.getProperties().toMap().get(DEFAULT_BASE_LOCATION_KEY); + String defaultBaseLocation = + externalCatalog.getProperties().toMap().get(DEFAULT_BASE_LOCATION_KEY); validateGsLocation("catalog.properties." + DEFAULT_BASE_LOCATION_KEY, defaultBaseLocation); List allowedLocations = gcpStorageConfigInfo.getAllowedLocations(); if (allowedLocations != null) { for (int index = 0; index < allowedLocations.size(); index++) { - validateGsLocation("storageConfigInfo.allowedLocations[" + index + "]", allowedLocations.get(index)); + validateGsLocation( + "storageConfigInfo.allowedLocations[" + index + "]", allowedLocations.get(index)); } } @@ -275,7 +278,8 @@ private static void validateGsLocation(String fieldName, String location) { "Invalid BigLake " + fieldName + " '" + location + "': malformed gs:// location.", e); } - if (!"gs".equalsIgnoreCase(storageUri.scheme()) || Strings.isNullOrEmpty(storageUri.authority())) { + if (!"gs".equalsIgnoreCase(storageUri.scheme()) + || Strings.isNullOrEmpty(storageUri.authority())) { throw new IllegalArgumentException( "Invalid BigLake " + fieldName + " '" + location + "': expected a gs:// location."); } diff --git a/runtime/service/src/test/java/org/apache/polaris/service/admin/BigLakeCatalogValidatorTest.java b/runtime/service/src/test/java/org/apache/polaris/service/admin/BigLakeCatalogValidatorTest.java index 322ea426393..c1bdc8c4d1d 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/admin/BigLakeCatalogValidatorTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/admin/BigLakeCatalogValidatorTest.java @@ -391,8 +391,7 @@ private Catalog bigLakeCatalog( String defaultBaseLocation, StorageConfigInfo storageConfigInfo) { CatalogProperties catalogProperties = CatalogProperties.builder(defaultBaseLocation).build(); - catalogProperties.put( - "enable.credential.vending", Boolean.toString(credentialVendingEnabled)); + catalogProperties.put("enable.credential.vending", Boolean.toString(credentialVendingEnabled)); return ExternalCatalog.builder() .setType(Catalog.TypeEnum.EXTERNAL) diff --git a/runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java b/runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java index 5877eebabd1..855b3c15688 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java @@ -371,7 +371,6 @@ public void testUpdateCatalogWithDisallowedConfigs() { "Explicitly setting polaris.config.enable-sub-catalog-rbac-for-federated-catalogs is not allowed because ALLOW_SETTING_SUB_CATALOG_RBAC_FOR_FEDERATED_CATALOGS is set to false."); } - @Test public void testCreateAndUpdateValidBigLakeCatalog() { String catalogName = "biglake-catalog"; @@ -379,9 +378,7 @@ public void testCreateAndUpdateValidBigLakeCatalog() { String updatedBaseLocation = "gs://bucket/path/to/updated-data"; Catalog catalog = createBigLakeCatalog( - catalogName, - initialBaseLocation, - createBigLakeStorageConfig(initialBaseLocation)); + catalogName, initialBaseLocation, createBigLakeStorageConfig(initialBaseLocation)); try (Response response = services @@ -418,10 +415,7 @@ public void testCreateAndUpdateValidBigLakeCatalog() { services .catalogsApi() .updateCatalog( - catalogName, - updateRequest, - services.realmContext(), - services.securityContext())) { + catalogName, updateRequest, services.realmContext(), services.securityContext())) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); Catalog updatedCatalog = (Catalog) response.getEntity(); assertThat(updatedCatalog.getProperties().getDefaultBaseLocation()) @@ -489,9 +483,7 @@ public void testUpdateBigLakeCatalogRejectsInvalidMergedConfiguration() { String initialBaseLocation = "gs://bucket/path/to/data"; Catalog catalog = createBigLakeCatalog( - catalogName, - initialBaseLocation, - createBigLakeStorageConfig(initialBaseLocation)); + catalogName, initialBaseLocation, createBigLakeStorageConfig(initialBaseLocation)); try (Response response = services @@ -532,7 +524,7 @@ public void testUpdateBigLakeCatalogRejectsInvalidMergedConfiguration() { updateRequest, services.realmContext(), services.securityContext())) - .isInstanceOf(IllegalArgumentException.class) + .isInstanceOfAny(BadRequestException.class, IllegalArgumentException.class) .hasMessageContaining("catalog.properties.default-base-location"); } @@ -558,7 +550,6 @@ private PolarisAdminService setupPolarisAdminService( ReservedProperties.NONE); } - private Catalog createBigLakeCatalog( String catalogName, String defaultBaseLocation, StorageConfigInfo storageConfigInfo) { CatalogProperties catalogProperties = CatalogProperties.builder(defaultBaseLocation).build();