Extend FileIOFactory to Improve Customization (apache#724)

* Initial Commit

* Fix test errors

* Resolved some comments

* Resolved some comments

* Removed unused package

* Make FileIOFactory as an ApplicationScoped bean

* Revert some changes

* Use FileIOFactory to get the creds for cleanup task

* Use new interface in tests

* Move some util classes to polaris-service-common as test fixtures

* Delete the interface in FileIOFactory that is only used for testing

* Refactor TestServices and use it in FileIOFactoryTest

* Small change

* Rename TestFileIOFactory to MeasuredFileIOFactory

* Use FileIOFactory to load catalog default FileIO in tests, fix log level for FileIOUtil

---------

Co-authored-by: Rulin Xing <rulin.xing@snowflake.com>

Author: XJDKC

quarkus/service/build.gradle.kts

@@ -90,6 +90,7 @@ dependencies {
   testFixturesApi(project(":polaris-tests"))
 
   testImplementation(project(":polaris-api-management-model"))
+  testImplementation(testFixtures(project(":polaris-service-common")))
 
   testImplementation("org.apache.iceberg:iceberg-api:${libs.versions.iceberg.get()}:tests")
   testImplementation("org.apache.iceberg:iceberg-core:${libs.versions.iceberg.get()}:tests")

quarkus/service/src/test/java/org/apache/polaris/service/quarkus/TestServices.java

@@ -32,15 +32,14 @@
 import org.apache.polaris.core.admin.model.PolarisCatalog;
 import org.apache.polaris.core.admin.model.StorageConfigInfo;
 import org.apache.polaris.core.admin.model.UpdateCatalogRequest;
-import org.apache.polaris.service.quarkus.TestServices;
-import org.apache.polaris.service.quarkus.catalog.io.TestFileIOFactory;
+import org.apache.polaris.service.TestServices;
 import org.junit.jupiter.api.Test;
 
 public class ManagementServiceTest {
   static TestServices services =
-      TestServices.inMemory(
-          new TestFileIOFactory(),
-          Map.of("SUPPORTED_CATALOG_STORAGE_TYPES", List.of("S3", "GCS", "AZURE")));
+      TestServices.builder()
+          .config(Map.of("SUPPORTED_CATALOG_STORAGE_TYPES", List.of("S3", "GCS", "AZURE")))
+          .build();
 
   @Test
   public void testCreateCatalogWithDisallowedStorageConfig() {

quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/ManagementServiceTest.java

@@ -67,10 +67,11 @@
 import org.apache.polaris.core.persistence.PolarisMetaStoreSession;
 import org.apache.polaris.service.admin.PolarisAdminService;
 import org.apache.polaris.service.catalog.BasePolarisCatalog;
+import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView;
 import org.apache.polaris.service.catalog.io.DefaultFileIOFactory;
+import org.apache.polaris.service.catalog.io.FileIOFactory;
 import org.apache.polaris.service.config.DefaultConfigurationStore;
 import org.apache.polaris.service.config.RealmEntityManagerFactory;
-import org.apache.polaris.service.quarkus.catalog.PolarisPassthroughResolutionView;
 import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl;
 import org.assertj.core.api.Assertions;
 import org.junit.jupiter.api.AfterEach;
@@ -160,6 +161,7 @@ public Map<String, String> getConfigOverrides() {
   protected PolarisEntityManager entityManager;
   protected PolarisMetaStoreManager metaStoreManager;
   protected PolarisMetaStoreSession metaStoreSession;
+  protected FileIOFactory fileIOFactory;
   protected PolarisBaseEntity catalogEntity;
   protected PrincipalEntity principalEntity;
   protected RealmId realmId;
@@ -385,6 +387,8 @@ private void initBaseCatalog() {
     PolarisPassthroughResolutionView passthroughView =
         new PolarisPassthroughResolutionView(
             entityManager, metaStoreSession, securityContext, CATALOG_NAME);
+    this.fileIOFactory =
+        new DefaultFileIOFactory(realmEntityManagerFactory, managerFactory, configurationStore);
     this.baseCatalog =
         new BasePolarisCatalog(
             realmId,
@@ -396,7 +400,7 @@ private void initBaseCatalog() {
             passthroughView,
             securityContext,
             Mockito.mock(),
-            new DefaultFileIOFactory());
+            fileIOFactory);
     this.baseCatalog.initialize(
         CATALOG_NAME,
         ImmutableMap.of(

quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java

@@ -33,16 +33,14 @@
 import org.apache.polaris.core.admin.model.CatalogProperties;
 import org.apache.polaris.core.admin.model.CreateCatalogRequest;
 import org.apache.polaris.core.admin.model.StorageConfigInfo;
-import org.apache.polaris.service.quarkus.TestServices;
-import org.apache.polaris.service.quarkus.catalog.io.TestFileIOFactory;
+import org.apache.polaris.service.TestServices;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.CsvSource;
 
 public class PolarisOverlappingCatalogTest {
 
   static TestServices services =
-      TestServices.inMemory(
-          new TestFileIOFactory(), Map.of("ALLOW_OVERLAPPING_CATALOG_URLS", "false"));
+      TestServices.builder().config(Map.of("ALLOW_OVERLAPPING_CATALOG_URLS", "false")).build();
 
   private Response createCatalog(String prefix, String defaultBaseLocation, boolean isExternal) {
     return createCatalog(prefix, defaultBaseLocation, isExternal, new ArrayList<String>());

quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisOverlappingCatalogTest.java

@@ -36,7 +36,7 @@
 import org.apache.polaris.core.admin.model.CreateCatalogRequest;
 import org.apache.polaris.core.admin.model.FileStorageConfigInfo;
 import org.apache.polaris.core.admin.model.StorageConfigInfo;
-import org.apache.polaris.service.quarkus.TestServices;
+import org.apache.polaris.service.TestServices;
 import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
@@ -105,7 +105,7 @@ void testTableLocationRestrictions(
       Map<String, Object> serverConfig,
       Map<String, String> catalogConfig,
       int expectedStatusForOverlaps) {
-    TestServices services = TestServices.inMemory(serverConfig);
+    TestServices services = TestServices.builder().config(serverConfig).build();
 
     CatalogProperties.Builder propertiesBuilder =
         CatalogProperties.builder()