From 4e1766bfbfd910e6748353a3fe9614eaa41cb830 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 25 Jun 2025 11:42:13 -0700 Subject: [PATCH 01/18] initial commit --- .../relational/jdbc/DatabaseType.java | 20 ++++++++- .../jdbc/JdbcMetaStoreManagerFactory.java | 45 +++++++++++++++---- .../persistence/MetaStoreManagerFactory.java | 8 +++- .../bootstrap/BootstrapOptions.java | 31 +++++++++++++ .../persistence/bootstrap/SchemaOptions.java | 28 ++++++++++++ .../polaris/admintool/BootstrapCommand.java | 40 ++++++++++++++++- 6 files changed, 157 insertions(+), 15 deletions(-) create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/BootstrapOptions.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/SchemaOptions.java diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java index b26247aad0..8a41b0221c 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java @@ -18,6 +18,9 @@ */ package org.apache.polaris.persistence.relational.jdbc; +import jakarta.annotation.Nonnull; +import org.apache.polaris.core.persistence.bootstrap.SchemaOptions; + import java.util.Locale; public enum DatabaseType { @@ -43,7 +46,20 @@ public static DatabaseType fromDisplayName(String displayName) { }; } - public String getInitScriptResource() { - return String.format("%s/schema-v1.sql", this.getDisplayName()); + public String getInitScriptResource(@Nonnull SchemaOptions schemaOptions) { + final String schemaFile; + if (!schemaOptions.schemaFile().isEmpty()) { + schemaFile = schemaOptions.schemaFile(); + } else { + String schemaSuffix; + switch (schemaOptions.schemaVersion()) { + case "LATEST" -> schemaSuffix = "schema-v1.sql"; + case "1" -> schemaSuffix = "schema-v1.sql"; + default -> + throw new IllegalArgumentException("Unknown schema version " + schemaOptions.schemaVersion()); + } + schemaFile = this.getDisplayName() + "/" + schemaSuffix; + } + return schemaFile; } } diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java index 118fcadd4f..556df637c2 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java @@ -26,6 +26,7 @@ import java.sql.SQLException; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.function.Supplier; import javax.sql.DataSource; import org.apache.polaris.core.PolarisCallContext; @@ -43,7 +44,11 @@ import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PrincipalSecretsGenerator; +import org.apache.polaris.core.persistence.bootstrap.BootstrapOptions; +import org.apache.polaris.core.persistence.bootstrap.ImmutableBootstrapOptions; +import org.apache.polaris.core.persistence.bootstrap.ImmutableSchemaOptions; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; +import org.apache.polaris.core.persistence.bootstrap.SchemaOptions; import org.apache.polaris.core.persistence.cache.EntityCache; import org.apache.polaris.core.persistence.cache.InMemoryEntityCache; import org.apache.polaris.core.persistence.dao.entity.BaseResult; @@ -93,8 +98,10 @@ protected PolarisMetaStoreManager createNewMetaStoreManager() { } private void initializeForRealm( - RealmContext realmContext, RootCredentialsSet rootCredentialsSet, boolean isBootstrap) { - DatasourceOperations databaseOperations = getDatasourceOperations(isBootstrap); + RealmContext realmContext, + RootCredentialsSet rootCredentialsSet, + Optional schemaOptions) { + DatasourceOperations databaseOperations = getDatasourceOperations(schemaOptions); sessionSupplierMap.put( realmContext.getRealmIdentifier(), () -> @@ -108,18 +115,19 @@ private void initializeForRealm( metaStoreManagerMap.put(realmContext.getRealmIdentifier(), metaStoreManager); } - private DatasourceOperations getDatasourceOperations(boolean isBootstrap) { + private DatasourceOperations getDatasourceOperations(Optional schemaOptions) { DatasourceOperations databaseOperations; try { databaseOperations = new DatasourceOperations(dataSource.get(), relationalJdbcConfiguration); } catch (SQLException sqlException) { throw new RuntimeException(sqlException); } - if (isBootstrap) { + // If this is set, we are bootstrapping + if (schemaOptions.isPresent()) { try { // Run the set-up script to create the tables. databaseOperations.executeScript( - databaseOperations.getDatabaseType().getInitScriptResource()); + databaseOperations.getDatabaseType().getInitScriptResource(schemaOptions.get())); } catch (SQLException e) { throw new RuntimeException( String.format("Error executing sql script: %s", e.getMessage()), e); @@ -131,12 +139,31 @@ private DatasourceOperations getDatasourceOperations(boolean isBootstrap) { @Override public synchronized Map bootstrapRealms( Iterable realms, RootCredentialsSet rootCredentialsSet) { + SchemaOptions schemaOptions = ImmutableSchemaOptions + .builder() + .build(); + + BootstrapOptions bootstrapOptions = ImmutableBootstrapOptions + .builder() + .realms(realms) + .rootCredentialsSet(rootCredentialsSet) + .schemaOptions(schemaOptions) + .build(); + + return bootstrapRealms(bootstrapOptions); + } + + @Override + public synchronized Map bootstrapRealms(BootstrapOptions bootstrapOptions) { Map results = new HashMap<>(); - for (String realm : realms) { + for (String realm : bootstrapOptions.realms()) { RealmContext realmContext = () -> realm; if (!metaStoreManagerMap.containsKey(realm)) { - initializeForRealm(realmContext, rootCredentialsSet, true); + initializeForRealm( + realmContext, + bootstrapOptions.rootCredentialsSet(), + Optional.of(bootstrapOptions.schemaOptions())); PrincipalSecretsResult secretsResult = bootstrapServiceAndCreatePolarisPrincipalForRealm( realmContext, metaStoreManagerMap.get(realm)); @@ -172,7 +199,7 @@ public Map purgeRealms(Iterable realms) { public synchronized PolarisMetaStoreManager getOrCreateMetaStoreManager( RealmContext realmContext) { if (!metaStoreManagerMap.containsKey(realmContext.getRealmIdentifier())) { - initializeForRealm(realmContext, null, false); + initializeForRealm(realmContext, null, Optional.empty()); checkPolarisServiceBootstrappedForRealm( realmContext, metaStoreManagerMap.get(realmContext.getRealmIdentifier())); } @@ -183,7 +210,7 @@ public synchronized PolarisMetaStoreManager getOrCreateMetaStoreManager( public synchronized Supplier getOrCreateSessionSupplier( RealmContext realmContext) { if (!sessionSupplierMap.containsKey(realmContext.getRealmIdentifier())) { - initializeForRealm(realmContext, null, false); + initializeForRealm(realmContext, null, Optional.empty()); checkPolarisServiceBootstrappedForRealm( realmContext, metaStoreManagerMap.get(realmContext.getRealmIdentifier())); } else { diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java index cb2523891f..c9da1ac73c 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java @@ -21,6 +21,7 @@ import java.util.Map; import java.util.function.Supplier; import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.persistence.bootstrap.BootstrapOptions; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; import org.apache.polaris.core.persistence.cache.EntityCache; import org.apache.polaris.core.persistence.dao.entity.BaseResult; @@ -38,8 +39,11 @@ public interface MetaStoreManagerFactory { EntityCache getOrCreateEntityCache(RealmContext realmContext); - Map bootstrapRealms( - Iterable realms, RootCredentialsSet rootCredentialsSet); + Map bootstrapRealms(Iterable realms, RootCredentialsSet rootCredentialsSet); + + default Map bootstrapRealms(BootstrapOptions bootstrapOptions) { + return bootstrapRealms(bootstrapOptions.realms(), bootstrapOptions.rootCredentialsSet()); + } /** Purge all metadata for the realms provided */ Map purgeRealms(Iterable realms); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/BootstrapOptions.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/BootstrapOptions.java new file mode 100644 index 0000000000..d217fcecb5 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/BootstrapOptions.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.core.persistence.bootstrap; + +import org.apache.polaris.immutables.PolarisImmutable; + +@PolarisImmutable +public interface BootstrapOptions { + Iterable realms(); + + RootCredentialsSet rootCredentialsSet(); + + SchemaOptions schemaOptions(); +} \ No newline at end of file diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/SchemaOptions.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/SchemaOptions.java new file mode 100644 index 0000000000..6f6b6b8adf --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/SchemaOptions.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.core.persistence.bootstrap; + +import org.apache.polaris.immutables.PolarisImmutable; + +@PolarisImmutable +public interface SchemaOptions { + String schemaVersion(); + String schemaFile(); +} diff --git a/runtime/admin/src/main/java/org/apache/polaris/admintool/BootstrapCommand.java b/runtime/admin/src/main/java/org/apache/polaris/admintool/BootstrapCommand.java index 8ddb350954..355545a39a 100644 --- a/runtime/admin/src/main/java/org/apache/polaris/admintool/BootstrapCommand.java +++ b/runtime/admin/src/main/java/org/apache/polaris/admintool/BootstrapCommand.java @@ -21,7 +21,12 @@ import java.nio.file.Path; import java.util.List; import java.util.Map; + +import org.apache.polaris.core.persistence.bootstrap.BootstrapOptions; +import org.apache.polaris.core.persistence.bootstrap.ImmutableBootstrapOptions; +import org.apache.polaris.core.persistence.bootstrap.ImmutableSchemaOptions; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; +import org.apache.polaris.core.persistence.bootstrap.SchemaOptions; import org.apache.polaris.core.persistence.dao.entity.PrincipalSecretsResult; import picocli.CommandLine; @@ -42,6 +47,9 @@ static class InputOptions { @CommandLine.ArgGroup(multiplicity = "1") FileInputOptions fileOptions; + @CommandLine.ArgGroup(multiplicity = "1") + SchemaInputOptions schemaInputOptions; + static class StandardInputOptions { @CommandLine.Option( @@ -71,6 +79,22 @@ static class FileInputOptions { description = "A file containing root principal credentials to bootstrap.") Path file; } + + static class SchemaInputOptions { + @CommandLine.Option( + names = {"-v", "--schema-version"}, + paramLabel = "", + description = "The version of the schema to load in [1, 2, LATEST].", + defaultValue = "LATEST") + String schemaVersion; + + @CommandLine.Option( + names = {"--schema-file"}, + paramLabel = "", + description = "A schema file to bootstrap from. If unset, the bundled files will be used.", + defaultValue = "") + String schemaFile; + } } @Override @@ -103,9 +127,21 @@ public Integer call() { } } + SchemaOptions schemaOptions = ImmutableSchemaOptions + .builder() + .schemaFile(inputOptions.schemaInputOptions.schemaFile) + .schemaVersion(inputOptions.schemaInputOptions.schemaVersion) + .build(); + + BootstrapOptions bootstrapOptions = ImmutableBootstrapOptions + .builder() + .realms(realms) + .rootCredentialsSet(rootCredentialsSet) + .schemaOptions(schemaOptions) + .build(); + // Execute the bootstrap - Map results = - metaStoreManagerFactory.bootstrapRealms(realms, rootCredentialsSet); + Map results = metaStoreManagerFactory.bootstrapRealms(bootstrapOptions); // Log any errors: boolean success = true; From 2fe7cdc56f1040361e4d8539c46dc952c63151dd Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 25 Jun 2025 11:50:44 -0700 Subject: [PATCH 02/18] stabilize some tests --- .../relational/jdbc/DatabaseType.java | 10 ++-- .../jdbc/JdbcMetaStoreManagerFactory.java | 25 +++++----- .../persistence/MetaStoreManagerFactory.java | 3 +- .../bootstrap/BootstrapOptions.java | 3 +- .../persistence/bootstrap/SchemaOptions.java | 11 ++++- .../polaris/admintool/BootstrapCommand.java | 48 +++++++++++-------- 6 files changed, 58 insertions(+), 42 deletions(-) diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java index 8a41b0221c..6efe15098e 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java @@ -19,9 +19,8 @@ package org.apache.polaris.persistence.relational.jdbc; import jakarta.annotation.Nonnull; -import org.apache.polaris.core.persistence.bootstrap.SchemaOptions; - import java.util.Locale; +import org.apache.polaris.core.persistence.bootstrap.SchemaOptions; public enum DatabaseType { POSTGRES("postgres"), @@ -48,15 +47,16 @@ public static DatabaseType fromDisplayName(String displayName) { public String getInitScriptResource(@Nonnull SchemaOptions schemaOptions) { final String schemaFile; - if (!schemaOptions.schemaFile().isEmpty()) { + if (schemaOptions.schemaFile() != null) { schemaFile = schemaOptions.schemaFile(); } else { String schemaSuffix; switch (schemaOptions.schemaVersion()) { - case "LATEST" -> schemaSuffix = "schema-v1.sql"; + case SchemaOptions.LATEST -> schemaSuffix = "schema-v1.sql"; case "1" -> schemaSuffix = "schema-v1.sql"; default -> - throw new IllegalArgumentException("Unknown schema version " + schemaOptions.schemaVersion()); + throw new IllegalArgumentException( + "Unknown schema version " + schemaOptions.schemaVersion()); } schemaFile = this.getDisplayName() + "/" + schemaSuffix; } diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java index 556df637c2..63027bcd28 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java @@ -139,31 +139,30 @@ private DatasourceOperations getDatasourceOperations(Optional sch @Override public synchronized Map bootstrapRealms( Iterable realms, RootCredentialsSet rootCredentialsSet) { - SchemaOptions schemaOptions = ImmutableSchemaOptions - .builder() - .build(); + SchemaOptions schemaOptions = ImmutableSchemaOptions.builder().build(); - BootstrapOptions bootstrapOptions = ImmutableBootstrapOptions - .builder() - .realms(realms) - .rootCredentialsSet(rootCredentialsSet) - .schemaOptions(schemaOptions) - .build(); + BootstrapOptions bootstrapOptions = + ImmutableBootstrapOptions.builder() + .realms(realms) + .rootCredentialsSet(rootCredentialsSet) + .schemaOptions(schemaOptions) + .build(); return bootstrapRealms(bootstrapOptions); } @Override - public synchronized Map bootstrapRealms(BootstrapOptions bootstrapOptions) { + public synchronized Map bootstrapRealms( + BootstrapOptions bootstrapOptions) { Map results = new HashMap<>(); for (String realm : bootstrapOptions.realms()) { RealmContext realmContext = () -> realm; if (!metaStoreManagerMap.containsKey(realm)) { initializeForRealm( - realmContext, - bootstrapOptions.rootCredentialsSet(), - Optional.of(bootstrapOptions.schemaOptions())); + realmContext, + bootstrapOptions.rootCredentialsSet(), + Optional.of(bootstrapOptions.schemaOptions())); PrincipalSecretsResult secretsResult = bootstrapServiceAndCreatePolarisPrincipalForRealm( realmContext, metaStoreManagerMap.get(realm)); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java index c9da1ac73c..5b5dadaa0f 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java @@ -39,7 +39,8 @@ public interface MetaStoreManagerFactory { EntityCache getOrCreateEntityCache(RealmContext realmContext); - Map bootstrapRealms(Iterable realms, RootCredentialsSet rootCredentialsSet); + Map bootstrapRealms( + Iterable realms, RootCredentialsSet rootCredentialsSet); default Map bootstrapRealms(BootstrapOptions bootstrapOptions) { return bootstrapRealms(bootstrapOptions.realms(), bootstrapOptions.rootCredentialsSet()); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/BootstrapOptions.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/BootstrapOptions.java index d217fcecb5..fea756c336 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/BootstrapOptions.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/BootstrapOptions.java @@ -19,6 +19,7 @@ package org.apache.polaris.core.persistence.bootstrap; +import jakarta.annotation.Nullable; import org.apache.polaris.immutables.PolarisImmutable; @PolarisImmutable @@ -28,4 +29,4 @@ public interface BootstrapOptions { RootCredentialsSet rootCredentialsSet(); SchemaOptions schemaOptions(); -} \ No newline at end of file +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/SchemaOptions.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/SchemaOptions.java index 6f6b6b8adf..b7bc37610a 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/SchemaOptions.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/SchemaOptions.java @@ -19,10 +19,19 @@ package org.apache.polaris.core.persistence.bootstrap; +import jakarta.annotation.Nullable; import org.apache.polaris.immutables.PolarisImmutable; +import org.immutables.value.Value; @PolarisImmutable public interface SchemaOptions { - String schemaVersion(); + public static final String LATEST = "LATEST"; + + @Value.Default + default String schemaVersion() { + return LATEST; + } + + @Nullable String schemaFile(); } diff --git a/runtime/admin/src/main/java/org/apache/polaris/admintool/BootstrapCommand.java b/runtime/admin/src/main/java/org/apache/polaris/admintool/BootstrapCommand.java index 355545a39a..32cd5de5c6 100644 --- a/runtime/admin/src/main/java/org/apache/polaris/admintool/BootstrapCommand.java +++ b/runtime/admin/src/main/java/org/apache/polaris/admintool/BootstrapCommand.java @@ -21,7 +21,6 @@ import java.nio.file.Path; import java.util.List; import java.util.Map; - import org.apache.polaris.core.persistence.bootstrap.BootstrapOptions; import org.apache.polaris.core.persistence.bootstrap.ImmutableBootstrapOptions; import org.apache.polaris.core.persistence.bootstrap.ImmutableSchemaOptions; @@ -82,17 +81,18 @@ static class FileInputOptions { static class SchemaInputOptions { @CommandLine.Option( - names = {"-v", "--schema-version"}, - paramLabel = "", - description = "The version of the schema to load in [1, 2, LATEST].", - defaultValue = "LATEST") + names = {"-v", "--schema-version"}, + paramLabel = "", + description = "The version of the schema to load in [1, 2, LATEST].", + defaultValue = SchemaOptions.LATEST) String schemaVersion; @CommandLine.Option( - names = {"--schema-file"}, - paramLabel = "", - description = "A schema file to bootstrap from. If unset, the bundled files will be used.", - defaultValue = "") + names = {"--schema-file"}, + paramLabel = "", + description = + "A schema file to bootstrap from. If unset, the bundled files will be used.", + defaultValue = "") String schemaFile; } } @@ -127,21 +127,27 @@ public Integer call() { } } - SchemaOptions schemaOptions = ImmutableSchemaOptions - .builder() - .schemaFile(inputOptions.schemaInputOptions.schemaFile) - .schemaVersion(inputOptions.schemaInputOptions.schemaVersion) - .build(); + final SchemaOptions schemaOptions; + if (inputOptions.schemaInputOptions != null) { + schemaOptions = + ImmutableSchemaOptions.builder() + .schemaFile(inputOptions.schemaInputOptions.schemaFile) + .schemaVersion(inputOptions.schemaInputOptions.schemaVersion) + .build(); + } else { + schemaOptions = ImmutableSchemaOptions.builder().build(); + } - BootstrapOptions bootstrapOptions = ImmutableBootstrapOptions - .builder() - .realms(realms) - .rootCredentialsSet(rootCredentialsSet) - .schemaOptions(schemaOptions) - .build(); + BootstrapOptions bootstrapOptions = + ImmutableBootstrapOptions.builder() + .realms(realms) + .rootCredentialsSet(rootCredentialsSet) + .schemaOptions(schemaOptions) + .build(); // Execute the bootstrap - Map results = metaStoreManagerFactory.bootstrapRealms(bootstrapOptions); + Map results = + metaStoreManagerFactory.bootstrapRealms(bootstrapOptions); // Log any errors: boolean success = true; From 3be2225278771c78a220172e7a23dd67caac0461 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 25 Jun 2025 11:52:20 -0700 Subject: [PATCH 03/18] autolint --- .../core/persistence/bootstrap/BootstrapOptions.java | 1 - .../org/apache/polaris/admintool/BootstrapCommand.java | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/BootstrapOptions.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/BootstrapOptions.java index fea756c336..a9bd907788 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/BootstrapOptions.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/BootstrapOptions.java @@ -19,7 +19,6 @@ package org.apache.polaris.core.persistence.bootstrap; -import jakarta.annotation.Nullable; import org.apache.polaris.immutables.PolarisImmutable; @PolarisImmutable diff --git a/runtime/admin/src/main/java/org/apache/polaris/admintool/BootstrapCommand.java b/runtime/admin/src/main/java/org/apache/polaris/admintool/BootstrapCommand.java index 32cd5de5c6..c80a7e6e27 100644 --- a/runtime/admin/src/main/java/org/apache/polaris/admintool/BootstrapCommand.java +++ b/runtime/admin/src/main/java/org/apache/polaris/admintool/BootstrapCommand.java @@ -130,10 +130,10 @@ public Integer call() { final SchemaOptions schemaOptions; if (inputOptions.schemaInputOptions != null) { schemaOptions = - ImmutableSchemaOptions.builder() - .schemaFile(inputOptions.schemaInputOptions.schemaFile) - .schemaVersion(inputOptions.schemaInputOptions.schemaVersion) - .build(); + ImmutableSchemaOptions.builder() + .schemaFile(inputOptions.schemaInputOptions.schemaFile) + .schemaVersion(inputOptions.schemaInputOptions.schemaVersion) + .build(); } else { schemaOptions = ImmutableSchemaOptions.builder().build(); } From 0a8c6d0cd73a0e144e3574709c9a80d1b0504394 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 25 Jun 2025 12:22:17 -0700 Subject: [PATCH 04/18] fix? --- .../apache/polaris/admintool/BootstrapCommandTestBase.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/BootstrapCommandTestBase.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/BootstrapCommandTestBase.java index 6ebbe0167f..aca46c2d2f 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/BootstrapCommandTestBase.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/BootstrapCommandTestBase.java @@ -91,8 +91,9 @@ public void testBootstrapInvalidCredentials(LaunchResult result) { public void testBootstrapInvalidArguments(LaunchResult result) { assertThat(result.getErrorOutput()) .contains( - "Error: (-r= [-r=]... [-c=]... [-p]) " - + "and -f= are mutually exclusive (specify only one)"); + "(-r= [-r=]... [-c=]... [-p]) and -f= " + + "and ([-v=] | [--schema-file=]) are mutually exclusive " + + "(specify only one)"); } @Test From 9969903275c2a820dc35b9ab1b8396f70f225136 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 25 Jun 2025 12:22:18 -0700 Subject: [PATCH 05/18] autolint --- .../apache/polaris/admintool/BootstrapCommandTestBase.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/BootstrapCommandTestBase.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/BootstrapCommandTestBase.java index aca46c2d2f..dab13c9148 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/BootstrapCommandTestBase.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/BootstrapCommandTestBase.java @@ -91,9 +91,9 @@ public void testBootstrapInvalidCredentials(LaunchResult result) { public void testBootstrapInvalidArguments(LaunchResult result) { assertThat(result.getErrorOutput()) .contains( - "(-r= [-r=]... [-c=]... [-p]) and -f= " + - "and ([-v=] | [--schema-file=]) are mutually exclusive " + - "(specify only one)"); + "(-r= [-r=]... [-c=]... [-p]) and -f= " + + "and ([-v=] | [--schema-file=]) are mutually exclusive " + + "(specify only one)"); } @Test From 6e9546ba69926ef07dc206d791343862fb70e8ba Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 25 Jun 2025 16:02:26 -0700 Subject: [PATCH 06/18] support arbitrary files --- .../relational/jdbc/DatabaseType.java | 20 +++++++++++++------ .../relational/jdbc/DatasourceOperations.java | 6 +++--- ...anagerWithJdbcBasePersistenceImplTest.java | 7 +++++-- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java index 6efe15098e..c901f4202f 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java @@ -19,6 +19,10 @@ package org.apache.polaris.persistence.relational.jdbc; import jakarta.annotation.Nonnull; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; import java.util.Locale; import org.apache.polaris.core.persistence.bootstrap.SchemaOptions; @@ -45,12 +49,15 @@ public static DatabaseType fromDisplayName(String displayName) { }; } - public String getInitScriptResource(@Nonnull SchemaOptions schemaOptions) { - final String schemaFile; + public InputStream getInitScriptResource(@Nonnull SchemaOptions schemaOptions) { if (schemaOptions.schemaFile() != null) { - schemaFile = schemaOptions.schemaFile(); + try{ + return new FileInputStream(schemaOptions.schemaFile()); + } catch (IOException e) { + throw new IllegalArgumentException("Unable to load file " + schemaOptions.schemaFile(), e); + } } else { - String schemaSuffix; + final String schemaSuffix; switch (schemaOptions.schemaVersion()) { case SchemaOptions.LATEST -> schemaSuffix = "schema-v1.sql"; case "1" -> schemaSuffix = "schema-v1.sql"; @@ -58,8 +65,9 @@ public String getInitScriptResource(@Nonnull SchemaOptions schemaOptions) { throw new IllegalArgumentException( "Unknown schema version " + schemaOptions.schemaVersion()); } - schemaFile = this.getDisplayName() + "/" + schemaSuffix; + ClassLoader classLoader = DatasourceOperations.class.getClassLoader(); + return classLoader.getResourceAsStream(this.getDisplayName() + "/" + schemaSuffix); } - return schemaFile; + } } diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperations.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperations.java index 7017f1919d..a254eab6f3 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperations.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperations.java @@ -24,6 +24,7 @@ import jakarta.annotation.Nonnull; import java.io.BufferedReader; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; import java.sql.Connection; import java.sql.PreparedStatement; @@ -80,15 +81,14 @@ DatabaseType getDatabaseType() { * @param scriptFilePath : Path of SQL script. * @throws SQLException : Exception while executing the script. */ - public void executeScript(String scriptFilePath) throws SQLException { - ClassLoader classLoader = DatasourceOperations.class.getClassLoader(); + public void executeScript(InputStream scriptInputStream) throws SQLException { runWithinTransaction( connection -> { try (Statement statement = connection.createStatement()) { BufferedReader reader = new BufferedReader( new InputStreamReader( - Objects.requireNonNull(classLoader.getResourceAsStream(scriptFilePath)), + Objects.requireNonNull(scriptInputStream), UTF_8)); StringBuilder sqlBuffer = new StringBuilder(); String line; diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java index fbcd3a958b..beb3626ab0 100644 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java @@ -20,6 +20,7 @@ import static org.apache.polaris.core.persistence.PrincipalSecretsGenerator.RANDOM_SECRETS; +import java.io.InputStream; import java.sql.SQLException; import java.time.ZoneId; import java.util.Optional; @@ -49,8 +50,10 @@ protected PolarisTestMetaStoreManager createPolarisTestMetaStoreManager() { try { datasourceOperations = new DatasourceOperations(createH2DataSource(), new H2JdbcConfiguration()); - datasourceOperations.executeScript( - String.format("%s/schema-v1.sql", DatabaseType.H2.getDisplayName())); + ClassLoader classLoader = DatasourceOperations.class.getClassLoader(); + InputStream scriptStream = + classLoader.getResourceAsStream(String.format("%s/schema-v1.sql", DatabaseType.H2.getDisplayName())); + datasourceOperations.executeScript(scriptStream); } catch (SQLException e) { throw new RuntimeException( String.format( From db199394f9114cd7fa92a3ee6f975c3866ef2fa8 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 25 Jun 2025 16:02:28 -0700 Subject: [PATCH 07/18] autolint --- .../polaris/persistence/relational/jdbc/DatabaseType.java | 4 +--- .../persistence/relational/jdbc/DatasourceOperations.java | 4 +--- ...AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java | 3 ++- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java index c901f4202f..dbf484ef8d 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java @@ -19,7 +19,6 @@ package org.apache.polaris.persistence.relational.jdbc; import jakarta.annotation.Nonnull; - import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; @@ -51,7 +50,7 @@ public static DatabaseType fromDisplayName(String displayName) { public InputStream getInitScriptResource(@Nonnull SchemaOptions schemaOptions) { if (schemaOptions.schemaFile() != null) { - try{ + try { return new FileInputStream(schemaOptions.schemaFile()); } catch (IOException e) { throw new IllegalArgumentException("Unable to load file " + schemaOptions.schemaFile(), e); @@ -68,6 +67,5 @@ public InputStream getInitScriptResource(@Nonnull SchemaOptions schemaOptions) { ClassLoader classLoader = DatasourceOperations.class.getClassLoader(); return classLoader.getResourceAsStream(this.getDisplayName() + "/" + schemaSuffix); } - } } diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperations.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperations.java index a254eab6f3..427a79c4cf 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperations.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperations.java @@ -87,9 +87,7 @@ public void executeScript(InputStream scriptInputStream) throws SQLException { try (Statement statement = connection.createStatement()) { BufferedReader reader = new BufferedReader( - new InputStreamReader( - Objects.requireNonNull(scriptInputStream), - UTF_8)); + new InputStreamReader(Objects.requireNonNull(scriptInputStream), UTF_8)); StringBuilder sqlBuffer = new StringBuilder(); String line; while ((line = reader.readLine()) != null) { diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java index beb3626ab0..7d0e6ae8e4 100644 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java @@ -52,7 +52,8 @@ protected PolarisTestMetaStoreManager createPolarisTestMetaStoreManager() { new DatasourceOperations(createH2DataSource(), new H2JdbcConfiguration()); ClassLoader classLoader = DatasourceOperations.class.getClassLoader(); InputStream scriptStream = - classLoader.getResourceAsStream(String.format("%s/schema-v1.sql", DatabaseType.H2.getDisplayName())); + classLoader.getResourceAsStream( + String.format("%s/schema-v1.sql", DatabaseType.H2.getDisplayName())); datasourceOperations.executeScript(scriptStream); } catch (SQLException e) { throw new RuntimeException( From 6fee80f14e4904bc4f43d1849db713d60cd00a2c Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 25 Jun 2025 16:07:19 -0700 Subject: [PATCH 08/18] attempt refactor around datasourceOperations --- .../jdbc/JdbcMetaStoreManagerFactory.java | 39 +++++++++---------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java index 63027bcd28..2d94d2eab5 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java @@ -98,15 +98,14 @@ protected PolarisMetaStoreManager createNewMetaStoreManager() { } private void initializeForRealm( + DatasourceOperations datasourceOperations, RealmContext realmContext, - RootCredentialsSet rootCredentialsSet, - Optional schemaOptions) { - DatasourceOperations databaseOperations = getDatasourceOperations(schemaOptions); + RootCredentialsSet rootCredentialsSet) { sessionSupplierMap.put( realmContext.getRealmIdentifier(), () -> new JdbcBasePersistenceImpl( - databaseOperations, + datasourceOperations, secretsGenerator(realmContext, rootCredentialsSet), storageIntegrationProvider, realmContext.getRealmIdentifier())); @@ -115,24 +114,13 @@ private void initializeForRealm( metaStoreManagerMap.put(realmContext.getRealmIdentifier(), metaStoreManager); } - private DatasourceOperations getDatasourceOperations(Optional schemaOptions) { + public DatasourceOperations getDatasourceOperations() { DatasourceOperations databaseOperations; try { databaseOperations = new DatasourceOperations(dataSource.get(), relationalJdbcConfiguration); } catch (SQLException sqlException) { throw new RuntimeException(sqlException); } - // If this is set, we are bootstrapping - if (schemaOptions.isPresent()) { - try { - // Run the set-up script to create the tables. - databaseOperations.executeScript( - databaseOperations.getDatabaseType().getInitScriptResource(schemaOptions.get())); - } catch (SQLException e) { - throw new RuntimeException( - String.format("Error executing sql script: %s", e.getMessage()), e); - } - } return databaseOperations; } @@ -159,10 +147,19 @@ public synchronized Map bootstrapRealms( for (String realm : bootstrapOptions.realms()) { RealmContext realmContext = () -> realm; if (!metaStoreManagerMap.containsKey(realm)) { + DatasourceOperations datasourceOperations = getDatasourceOperations(); + try { + // Run the set-up script to create the tables. + datasourceOperations.executeScript( + datasourceOperations.getDatabaseType().getInitScriptResource(bootstrapOptions.schemaOptions())); + } catch (SQLException e) { + throw new RuntimeException( + String.format("Error executing sql script: %s", e.getMessage()), e); + } initializeForRealm( + datasourceOperations, realmContext, - bootstrapOptions.rootCredentialsSet(), - Optional.of(bootstrapOptions.schemaOptions())); + bootstrapOptions.rootCredentialsSet()); PrincipalSecretsResult secretsResult = bootstrapServiceAndCreatePolarisPrincipalForRealm( realmContext, metaStoreManagerMap.get(realm)); @@ -198,7 +195,8 @@ public Map purgeRealms(Iterable realms) { public synchronized PolarisMetaStoreManager getOrCreateMetaStoreManager( RealmContext realmContext) { if (!metaStoreManagerMap.containsKey(realmContext.getRealmIdentifier())) { - initializeForRealm(realmContext, null, Optional.empty()); + DatasourceOperations datasourceOperations = getDatasourceOperations(); + initializeForRealm(datasourceOperations, realmContext, null); checkPolarisServiceBootstrappedForRealm( realmContext, metaStoreManagerMap.get(realmContext.getRealmIdentifier())); } @@ -209,7 +207,8 @@ public synchronized PolarisMetaStoreManager getOrCreateMetaStoreManager( public synchronized Supplier getOrCreateSessionSupplier( RealmContext realmContext) { if (!sessionSupplierMap.containsKey(realmContext.getRealmIdentifier())) { - initializeForRealm(realmContext, null, Optional.empty()); + DatasourceOperations datasourceOperations = getDatasourceOperations(); + initializeForRealm(datasourceOperations, realmContext, null); checkPolarisServiceBootstrappedForRealm( realmContext, metaStoreManagerMap.get(realmContext.getRealmIdentifier())); } else { From 183cfbd67d1488eec4d2a5ccbbd90a40906d601e Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 25 Jun 2025 16:07:20 -0700 Subject: [PATCH 09/18] autolint --- .../relational/jdbc/JdbcMetaStoreManagerFactory.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java index 2d94d2eab5..f0ab4e0e62 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java @@ -26,7 +26,6 @@ import java.sql.SQLException; import java.util.HashMap; import java.util.Map; -import java.util.Optional; import java.util.function.Supplier; import javax.sql.DataSource; import org.apache.polaris.core.PolarisCallContext; @@ -151,15 +150,15 @@ public synchronized Map bootstrapRealms( try { // Run the set-up script to create the tables. datasourceOperations.executeScript( - datasourceOperations.getDatabaseType().getInitScriptResource(bootstrapOptions.schemaOptions())); + datasourceOperations + .getDatabaseType() + .getInitScriptResource(bootstrapOptions.schemaOptions())); } catch (SQLException e) { throw new RuntimeException( - String.format("Error executing sql script: %s", e.getMessage()), e); + String.format("Error executing sql script: %s", e.getMessage()), e); } initializeForRealm( - datasourceOperations, - realmContext, - bootstrapOptions.rootCredentialsSet()); + datasourceOperations, realmContext, bootstrapOptions.rootCredentialsSet()); PrincipalSecretsResult secretsResult = bootstrapServiceAndCreatePolarisPrincipalForRealm( realmContext, metaStoreManagerMap.get(realm)); From 1b53b8e4a9dfaf0b3302a5d2f6c0e115c4a340fc Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 25 Jun 2025 17:13:12 -0700 Subject: [PATCH 10/18] no more LATEST default --- .../persistence/relational/jdbc/DatabaseType.java | 3 ++- .../core/persistence/bootstrap/SchemaOptions.java | 13 +++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java index dbf484ef8d..6545103c94 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java @@ -55,9 +55,10 @@ public InputStream getInitScriptResource(@Nonnull SchemaOptions schemaOptions) { } catch (IOException e) { throw new IllegalArgumentException("Unable to load file " + schemaOptions.schemaFile(), e); } - } else { + } else if { final String schemaSuffix; switch (schemaOptions.schemaVersion()) { + case null -> schemaSuffix = "schema-v1.sql"; case SchemaOptions.LATEST -> schemaSuffix = "schema-v1.sql"; case "1" -> schemaSuffix = "schema-v1.sql"; default -> diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/SchemaOptions.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/SchemaOptions.java index b7bc37610a..268c8ba031 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/SchemaOptions.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/SchemaOptions.java @@ -27,11 +27,16 @@ public interface SchemaOptions { public static final String LATEST = "LATEST"; - @Value.Default - default String schemaVersion() { - return LATEST; - } + @Nullable + String schemaVersion(); @Nullable String schemaFile(); + + @Value.Check + default void validate() { + if (schemaVersion() != null && schemaFile() != null) { + throw new IllegalStateException("Only one of schemaVersion or schemaFile can be set."); + } + } } From 6a929e2bc44f330b70aca3c53a0e09d1f7c3ff54 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 25 Jun 2025 17:13:57 -0700 Subject: [PATCH 11/18] typofix --- .../polaris/persistence/relational/jdbc/DatabaseType.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java index 6545103c94..12929126ca 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java @@ -55,7 +55,7 @@ public InputStream getInitScriptResource(@Nonnull SchemaOptions schemaOptions) { } catch (IOException e) { throw new IllegalArgumentException("Unable to load file " + schemaOptions.schemaFile(), e); } - } else if { + } else { final String schemaSuffix; switch (schemaOptions.schemaVersion()) { case null -> schemaSuffix = "schema-v1.sql"; From 06e2a517fd5834e6d6da765e2f6ea901ba358fb0 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 25 Jun 2025 17:20:21 -0700 Subject: [PATCH 12/18] fix check --- .../polaris/core/persistence/bootstrap/SchemaOptions.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/SchemaOptions.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/SchemaOptions.java index 268c8ba031..80c308e92a 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/SchemaOptions.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/SchemaOptions.java @@ -38,5 +38,13 @@ default void validate() { if (schemaVersion() != null && schemaFile() != null) { throw new IllegalStateException("Only one of schemaVersion or schemaFile can be set."); } + if (schemaVersion() != null && !schemaVersion().equals(LATEST)) { + try { + Integer.parseInt(schemaVersion()); + } catch (NumberFormatException e) { + throw new IllegalArgumentException( + "schemaVersion must be a valid integer or " + LATEST + ": " + schemaVersion(), e); + } + } } } From b80f6aec2cfee2fc459199e4d59d98f481fac8b8 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Thu, 26 Jun 2025 14:21:22 -0700 Subject: [PATCH 13/18] try removing LATEST --- .../persistence/relational/jdbc/DatabaseType.java | 3 +-- .../core/persistence/bootstrap/SchemaOptions.java | 12 +----------- .../apache/polaris/admintool/BootstrapCommand.java | 5 ++--- 3 files changed, 4 insertions(+), 16 deletions(-) diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java index 12929126ca..ef8fd86fd3 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java @@ -59,8 +59,7 @@ public InputStream getInitScriptResource(@Nonnull SchemaOptions schemaOptions) { final String schemaSuffix; switch (schemaOptions.schemaVersion()) { case null -> schemaSuffix = "schema-v1.sql"; - case SchemaOptions.LATEST -> schemaSuffix = "schema-v1.sql"; - case "1" -> schemaSuffix = "schema-v1.sql"; + case 1 -> schemaSuffix = "schema-v1.sql"; default -> throw new IllegalArgumentException( "Unknown schema version " + schemaOptions.schemaVersion()); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/SchemaOptions.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/SchemaOptions.java index 80c308e92a..f0779cc8e1 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/SchemaOptions.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/SchemaOptions.java @@ -25,10 +25,8 @@ @PolarisImmutable public interface SchemaOptions { - public static final String LATEST = "LATEST"; - @Nullable - String schemaVersion(); + Integer schemaVersion(); @Nullable String schemaFile(); @@ -38,13 +36,5 @@ default void validate() { if (schemaVersion() != null && schemaFile() != null) { throw new IllegalStateException("Only one of schemaVersion or schemaFile can be set."); } - if (schemaVersion() != null && !schemaVersion().equals(LATEST)) { - try { - Integer.parseInt(schemaVersion()); - } catch (NumberFormatException e) { - throw new IllegalArgumentException( - "schemaVersion must be a valid integer or " + LATEST + ": " + schemaVersion(), e); - } - } } } diff --git a/runtime/admin/src/main/java/org/apache/polaris/admintool/BootstrapCommand.java b/runtime/admin/src/main/java/org/apache/polaris/admintool/BootstrapCommand.java index c80a7e6e27..2d75b4c17d 100644 --- a/runtime/admin/src/main/java/org/apache/polaris/admintool/BootstrapCommand.java +++ b/runtime/admin/src/main/java/org/apache/polaris/admintool/BootstrapCommand.java @@ -83,9 +83,8 @@ static class SchemaInputOptions { @CommandLine.Option( names = {"-v", "--schema-version"}, paramLabel = "", - description = "The version of the schema to load in [1, 2, LATEST].", - defaultValue = SchemaOptions.LATEST) - String schemaVersion; + description = "The version of the schema to load in [1, 2, LATEST].") + Integer schemaVersion; @CommandLine.Option( names = {"--schema-file"}, From 49cce5b333dde0936a9086edd861b2699264c534 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Thu, 26 Jun 2025 21:46:55 -0700 Subject: [PATCH 14/18] test fix --- .../org/apache/polaris/admintool/BootstrapCommandTestBase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/BootstrapCommandTestBase.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/BootstrapCommandTestBase.java index dab13c9148..1bb2543052 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/BootstrapCommandTestBase.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/BootstrapCommandTestBase.java @@ -92,7 +92,7 @@ public void testBootstrapInvalidArguments(LaunchResult result) { assertThat(result.getErrorOutput()) .contains( "(-r= [-r=]... [-c=]... [-p]) and -f= " - + "and ([-v=] | [--schema-file=]) are mutually exclusive " + + "and (-v= | [--schema-file=]) are mutually exclusive " + "(specify only one)"); } From e4a2f2aa693403871cb22dac4a025ec5cfc435df Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Fri, 27 Jun 2025 18:19:47 -0700 Subject: [PATCH 15/18] add close --- .../relational/jdbc/DatasourceOperations.java | 66 +++++++++++-------- 1 file changed, 37 insertions(+), 29 deletions(-) diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperations.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperations.java index 427a79c4cf..297b0e8803 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperations.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperations.java @@ -76,43 +76,51 @@ DatabaseType getDatabaseType() { } /** - * Execute SQL script. + * Execute SQL script and close the associated input stream * - * @param scriptFilePath : Path of SQL script. + * @param scriptInputStream : Input stream containing the SQL script. * @throws SQLException : Exception while executing the script. */ public void executeScript(InputStream scriptInputStream) throws SQLException { - runWithinTransaction( - connection -> { - try (Statement statement = connection.createStatement()) { - BufferedReader reader = - new BufferedReader( - new InputStreamReader(Objects.requireNonNull(scriptInputStream), UTF_8)); - StringBuilder sqlBuffer = new StringBuilder(); - String line; - while ((line = reader.readLine()) != null) { - line = line.trim(); - if (!line.isEmpty() && !line.startsWith("--")) { // Ignore empty lines and comments - sqlBuffer.append(line).append("\n"); - if (line.endsWith(";")) { // Execute statement when semicolon is found - String sql = sqlBuffer.toString().trim(); - try { - // since SQL is directly read from the file, there is close to 0 possibility - // of this being injected plus this run via an Admin tool, if attacker can - // fiddle with this that means lot of other things are already compromised. - statement.execute(sql); - } catch (SQLException e) { - throw new RuntimeException(e); + try { + runWithinTransaction( + connection -> { + try (Statement statement = connection.createStatement()) { + BufferedReader reader = + new BufferedReader( + new InputStreamReader(Objects.requireNonNull(scriptInputStream), UTF_8)); + StringBuilder sqlBuffer = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + line = line.trim(); + if (!line.isEmpty() && !line.startsWith("--")) { // Ignore empty lines and comments + sqlBuffer.append(line).append("\n"); + if (line.endsWith(";")) { // Execute statement when semicolon is found + String sql = sqlBuffer.toString().trim(); + try { + // since SQL is directly read from the file, there is close to 0 possibility + // of this being injected plus this run via an Admin tool, if attacker can + // fiddle with this that means lot of other things are already compromised. + statement.execute(sql); + } catch (SQLException e) { + throw new RuntimeException(e); + } + sqlBuffer.setLength(0); // Clear the buffer for the next statement } - sqlBuffer.setLength(0); // Clear the buffer for the next statement } } + return true; + } catch (IOException e) { + throw new RuntimeException(e); } - return true; - } catch (IOException e) { - throw new RuntimeException(e); - } - }); + }); + } finally { + try { + scriptInputStream.close(); + } catch (IOException e) { + LOGGER.error("Failed to close input stream: {}", e.getMessage()); + } + } } /** From 35592345684881a57fef03eb8497cf115d164a64 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 30 Jun 2025 14:03:57 -0700 Subject: [PATCH 16/18] rename --- .../polaris/persistence/relational/jdbc/DatabaseType.java | 6 +++++- .../relational/jdbc/JdbcMetaStoreManagerFactory.java | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java index 1f665c7d0f..011207ec99 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java @@ -48,7 +48,11 @@ public static DatabaseType fromDisplayName(String displayName) { }; } - public InputStream getInitScriptResource(@Nonnull SchemaOptions schemaOptions) { + /** + * Open an InputStream that contains data from an init script. This stream should be + * closed by the caller. + */ + public InputStream openInitScriptResource(@Nonnull SchemaOptions schemaOptions) { if (schemaOptions.schemaFile() != null) { try { return new FileInputStream(schemaOptions.schemaFile()); diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java index f0ab4e0e62..a69410a94f 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java @@ -152,7 +152,7 @@ public synchronized Map bootstrapRealms( datasourceOperations.executeScript( datasourceOperations .getDatabaseType() - .getInitScriptResource(bootstrapOptions.schemaOptions())); + .openInitScriptResource(bootstrapOptions.schemaOptions())); } catch (SQLException e) { throw new RuntimeException( String.format("Error executing sql script: %s", e.getMessage()), e); From 28617e0bf0e75c3c9b3f157d25b306603ef76cdb Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 30 Jun 2025 14:04:06 -0700 Subject: [PATCH 17/18] autolint --- .../polaris/persistence/relational/jdbc/DatabaseType.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java index 011207ec99..f731eb5508 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java @@ -49,8 +49,8 @@ public static DatabaseType fromDisplayName(String displayName) { } /** - * Open an InputStream that contains data from an init script. This stream should be - * closed by the caller. + * Open an InputStream that contains data from an init script. This stream should be closed by the + * caller. */ public InputStream openInitScriptResource(@Nonnull SchemaOptions schemaOptions) { if (schemaOptions.schemaFile() != null) { From b7e2e5a423607f36c2bef3d68aee998201dd40de Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 30 Jun 2025 14:11:39 -0700 Subject: [PATCH 18/18] try-with-resource --- .../persistence/relational/jdbc/DatasourceOperations.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperations.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperations.java index 297b0e8803..43d57ead59 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperations.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperations.java @@ -85,10 +85,10 @@ public void executeScript(InputStream scriptInputStream) throws SQLException { try { runWithinTransaction( connection -> { - try (Statement statement = connection.createStatement()) { - BufferedReader reader = - new BufferedReader( - new InputStreamReader(Objects.requireNonNull(scriptInputStream), UTF_8)); + try (Statement statement = connection.createStatement(); + BufferedReader reader = + new BufferedReader( + new InputStreamReader(Objects.requireNonNull(scriptInputStream), UTF_8))) { StringBuilder sqlBuffer = new StringBuilder(); String line; while ((line = reader.readLine()) != null) {