Add support for loading configuration from toml files

trinodb · Aug 8, 2024 · b2f71a3 · b2f71a3
1 parent 1301057
commit b2f71a3
Show file tree

Hide file tree

Showing 20 changed files with 215 additions and 70 deletions.
diff --git a/core/trino-main/src/main/java/io/trino/connector/CatalogStoreManager.java b/core/trino-main/src/main/java/io/trino/connector/CatalogStoreManager.java
@@ -25,8 +25,8 @@
 import io.trino.spi.classloader.ThreadContextClassLoader;
 import io.trino.spi.connector.ConnectorName;
 
-import java.io.File;
 import java.io.IOException;
+import java.nio.file.Path;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
@@ -37,14 +37,16 @@
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.base.Strings.isNullOrEmpty;
-import static io.airlift.configuration.ConfigurationLoader.loadPropertiesFrom;
+import static io.trino.plugin.base.ConfigurationLoader.configurationExists;
+import static io.trino.plugin.base.ConfigurationLoader.loadConfigurationFrom;
+import static io.trino.plugin.base.ConfigurationLoader.resolvedConfigurationPath;
 import static java.util.Objects.requireNonNull;
 
 public class CatalogStoreManager
         implements CatalogStore
 {
     private static final Logger log = Logger.get(CatalogStoreManager.class);
-    private static final File CATALOG_STORE_CONFIGURATION = new File("etc/catalog-store.properties");
+    private static final Path CATALOG_STORE_CONFIGURATION = Path.of("etc/catalog-store");
     private static final String CATALOG_STORE_PROPERTY_NAME = "catalog-store.name";
     private final Map<String, CatalogStoreFactory> catalogStoreFactories = new ConcurrentHashMap<>();
     private final AtomicReference<Optional<CatalogStore>> configuredCatalogStore = new AtomicReference<>(Optional.empty());
@@ -72,17 +74,17 @@ public void loadConfiguredCatalogStore()
     }
 
     @VisibleForTesting
-    void loadConfiguredCatalogStore(File catalogStoreFile)
+    void loadConfiguredCatalogStore(Path catalogStoreFile)
             throws IOException
     {
-        if (configuredCatalogStore.get().isPresent() || !catalogStoreFile.exists()) {
+        if (configuredCatalogStore.get().isPresent() || !configurationExists(catalogStoreFile)) {
             return;
         }
-        Map<String, String> properties = new HashMap<>(loadPropertiesFrom(catalogStoreFile.getPath()));
+        Map<String, String> properties = new HashMap<>(loadConfigurationFrom(catalogStoreFile));
 
         String catalogStoreName = properties.remove(CATALOG_STORE_PROPERTY_NAME);
         checkArgument(!isNullOrEmpty(catalogStoreName),
-                "Catalog store configuration %s does not contain %s", catalogStoreFile.getAbsoluteFile(), CATALOG_STORE_PROPERTY_NAME);
+                "Catalog store configuration %s does not contain %s", resolvedConfigurationPath(catalogStoreFile), CATALOG_STORE_PROPERTY_NAME);
 
         setConfiguredCatalogStore(catalogStoreName, properties);
     }

diff --git a/core/trino-main/src/main/java/io/trino/connector/FileCatalogStore.java b/core/trino-main/src/main/java/io/trino/connector/FileCatalogStore.java
@@ -21,6 +21,7 @@
 import com.google.inject.Inject;
 import io.airlift.log.Logger;
 import io.trino.connector.system.GlobalSystemConnector;
+import io.trino.plugin.base.ConfigurationLoader;
 import io.trino.spi.TrinoException;
 import io.trino.spi.catalog.CatalogName;
 import io.trino.spi.catalog.CatalogProperties;
@@ -48,7 +49,8 @@
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.io.Files.getNameWithoutExtension;
-import static io.airlift.configuration.ConfigurationLoader.loadPropertiesFrom;
+import static io.trino.plugin.base.ConfigurationLoader.PROPERTIES_SUFFIX;
+import static io.trino.plugin.base.ConfigurationLoader.loadConfigurationFrom;
 import static io.trino.spi.StandardErrorCode.CATALOG_STORE_ERROR;
 import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED;
 import static io.trino.spi.connector.CatalogHandle.createRootCatalogHandle;
@@ -151,7 +153,7 @@ private void checkModifiable()
 
     private File toFile(CatalogName catalogName)
     {
-        return new File(catalogsDirectory, catalogName.toString() + ".properties");
+        return new File(catalogsDirectory, catalogName.toString() + PROPERTIES_SUFFIX);
     }
 
     private static List<File> listCatalogFiles(File catalogsDirectory)
@@ -166,7 +168,7 @@ private static List<File> listCatalogFiles(File catalogsDirectory)
         }
         return Arrays.stream(files)
                 .filter(File::isFile)
-                .filter(file -> file.getName().endsWith(".properties"))
+                .filter(ConfigurationLoader::isConfigurationFile)
                 .collect(toImmutableList());
     }
 
@@ -217,7 +219,7 @@ public CatalogProperties loadProperties()
         {
             Map<String, String> properties;
             try {
-                properties = new HashMap<>(loadPropertiesFrom(file.getPath()));
+                properties = new HashMap<>(loadConfigurationFrom(file));
             }
             catch (IOException e) {
                 throw new UncheckedIOException("Error reading catalog property file " + file, e);

diff --git a/core/trino-main/src/main/java/io/trino/connector/StaticCatalogManager.java b/core/trino-main/src/main/java/io/trino/connector/StaticCatalogManager.java
@@ -24,6 +24,7 @@
 import io.trino.connector.system.GlobalSystemConnector;
 import io.trino.metadata.Catalog;
 import io.trino.metadata.CatalogManager;
+import io.trino.plugin.base.ConfigurationLoader;
 import io.trino.server.ForStartup;
 import io.trino.spi.TrinoException;
 import io.trino.spi.catalog.CatalogName;
@@ -52,7 +53,7 @@
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.collect.ImmutableList.toImmutableList;
-import static io.airlift.configuration.ConfigurationLoader.loadPropertiesFrom;
+import static io.trino.plugin.base.ConfigurationLoader.loadConfigurationFrom;
 import static io.trino.spi.StandardErrorCode.CATALOG_NOT_AVAILABLE;
 import static io.trino.spi.StandardErrorCode.CATALOG_NOT_FOUND;
 import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED;
@@ -93,7 +94,7 @@ public StaticCatalogManager(CatalogFactory catalogFactory, StaticCatalogManagerC
 
             Map<String, String> properties;
             try {
-                properties = new HashMap<>(loadPropertiesFrom(file.getPath()));
+                properties = new HashMap<>(loadConfigurationFrom(file));
             }
             catch (IOException e) {
                 throw new UncheckedIOException("Error reading catalog property file " + file, e);
@@ -128,7 +129,7 @@ private static List<File> listCatalogFiles(File catalogsDirectory)
         }
         return Arrays.stream(files)
                 .filter(File::isFile)
-                .filter(file -> file.getName().endsWith(".properties"))
+                .filter(ConfigurationLoader::isConfigurationFile)
                 .collect(toImmutableList());
     }
 

diff --git a/core/trino-main/src/main/java/io/trino/eventlistener/EventListenerManager.java b/core/trino-main/src/main/java/io/trino/eventlistener/EventListenerManager.java
@@ -33,6 +33,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -47,15 +48,17 @@
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.base.Strings.isNullOrEmpty;
 import static com.google.common.collect.ImmutableList.toImmutableList;
-import static io.airlift.configuration.ConfigurationLoader.loadPropertiesFrom;
+import static io.trino.plugin.base.ConfigurationLoader.configurationExists;
+import static io.trino.plugin.base.ConfigurationLoader.loadConfigurationFrom;
+import static io.trino.plugin.base.ConfigurationLoader.resolvedConfigurationPath;
 import static java.lang.String.format;
 import static java.util.Objects.requireNonNull;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 
 public class EventListenerManager
 {
     private static final Logger log = Logger.get(EventListenerManager.class);
-    private static final File CONFIG_FILE = new File("etc/event-listener.properties");
+    private static final Path CONFIG_FILE = Path.of("etc/event-listener");
     private static final String EVENT_LISTENER_NAME_PROPERTY = "event-listener.name";
     private final List<File> configFiles;
     private final int maxConcurrentQueryCompletedEvents;
@@ -109,10 +112,10 @@ private List<EventListener> configuredEventListeners()
     {
         List<File> configFiles = this.configFiles;
         if (configFiles.isEmpty()) {
-            if (!CONFIG_FILE.exists()) {
+            if (!configurationExists(CONFIG_FILE)) {
                 return ImmutableList.of();
             }
-            configFiles = ImmutableList.of(CONFIG_FILE);
+            configFiles = ImmutableList.of(resolvedConfigurationPath(CONFIG_FILE).toFile());
         }
         return configFiles.stream()
                 .map(this::createEventListener)
@@ -143,7 +146,7 @@ private EventListener createEventListener(File configFile)
     private static Map<String, String> loadEventListenerProperties(File configFile)
     {
         try {
-            return new HashMap<>(loadPropertiesFrom(configFile.getPath()));
+            return new HashMap<>(loadConfigurationFrom(configFile.getPath()));
         }
         catch (IOException e) {
             throw new UncheckedIOException("Failed to read configuration file: " + configFile, e);

diff --git a/core/trino-main/src/main/java/io/trino/exchange/ExchangeManagerRegistry.java b/core/trino-main/src/main/java/io/trino/exchange/ExchangeManagerRegistry.java
@@ -24,17 +24,18 @@
 import io.trino.spi.exchange.ExchangeManagerFactory;
 import jakarta.annotation.PreDestroy;
 
-import java.io.File;
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.nio.file.Path;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.base.Strings.isNullOrEmpty;
-import static io.airlift.configuration.ConfigurationLoader.loadPropertiesFrom;
+import static io.trino.plugin.base.ConfigurationLoader.configurationExists;
+import static io.trino.plugin.base.ConfigurationLoader.loadConfigurationFrom;
 import static io.trino.spi.StandardErrorCode.EXCHANGE_MANAGER_NOT_CONFIGURED;
 import static java.lang.String.format;
 import static java.util.Objects.requireNonNull;
@@ -43,7 +44,7 @@ public class ExchangeManagerRegistry
 {
     private static final Logger log = Logger.get(ExchangeManagerRegistry.class);
 
-    private static final File CONFIG_FILE = new File("etc/exchange-manager.properties");
+    private static final Path CONFIG_FILE = Path.of("etc/exchange-manager");
     private static final String EXCHANGE_MANAGER_NAME_PROPERTY = "exchange-manager.name";
 
     private final OpenTelemetry openTelemetry;
@@ -74,7 +75,7 @@ public void addExchangeManagerFactory(ExchangeManagerFactory factory)
 
     public void loadExchangeManager()
     {
-        if (!CONFIG_FILE.exists()) {
+        if (!configurationExists(CONFIG_FILE)) {
             return;
         }
 
@@ -126,10 +127,10 @@ public void shutdown()
         }
     }
 
-    private static Map<String, String> loadProperties(File configFile)
+    private static Map<String, String> loadProperties(Path configFile)
     {
         try {
-            return new HashMap<>(loadPropertiesFrom(configFile.getPath()));
+            return new HashMap<>(loadConfigurationFrom(configFile));
         }
         catch (IOException e) {
             throw new UncheckedIOException("Failed to read configuration file: " + configFile, e);

diff --git a/...no-main/src/main/java/io/trino/execution/resourcegroups/InternalResourceGroupManager.java b/...no-main/src/main/java/io/trino/execution/resourcegroups/InternalResourceGroupManager.java
@@ -37,7 +37,7 @@
 import org.weakref.jmx.MBeanExporter;
 import org.weakref.jmx.Managed;
 
-import java.io.File;
+import java.nio.file.Path;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -55,7 +55,9 @@
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.base.Strings.isNullOrEmpty;
 import static io.airlift.concurrent.Threads.daemonThreadsNamed;
-import static io.airlift.configuration.ConfigurationLoader.loadPropertiesFrom;
+import static io.trino.plugin.base.ConfigurationLoader.configurationExists;
+import static io.trino.plugin.base.ConfigurationLoader.loadConfigurationFrom;
+import static io.trino.plugin.base.ConfigurationLoader.resolvedConfigurationPath;
 import static io.trino.spi.StandardErrorCode.QUERY_REJECTED;
 import static java.lang.String.format;
 import static java.util.Objects.requireNonNull;
@@ -68,7 +70,7 @@ public final class InternalResourceGroupManager<C>
 {
     private static final Logger log = Logger.get(InternalResourceGroupManager.class);
 
-    private static final File CONFIG_FILE = new File("etc/resource-groups.properties");
+    private static final Path CONFIG_FILE = Path.of("etc/resource-groups");
     private static final String NAME_PROPERTY = "resource-groups.configuration-manager";
 
     private final ScheduledExecutorService refreshExecutor = newSingleThreadScheduledExecutor(daemonThreadsNamed("ResourceGroupManager"));
@@ -141,15 +143,14 @@ public void addConfigurationManagerFactory(ResourceGroupConfigurationManagerFact
     public void loadConfigurationManager()
             throws Exception
     {
-        File configFile = CONFIG_FILE.getAbsoluteFile();
-        if (!configFile.exists()) {
+        if (!configurationExists(CONFIG_FILE)) {
             return;
         }
 
-        Map<String, String> properties = new HashMap<>(loadPropertiesFrom(configFile.getPath()));
+        Map<String, String> properties = new HashMap<>(loadConfigurationFrom(CONFIG_FILE));
 
         String name = properties.remove(NAME_PROPERTY);
-        checkState(!isNullOrEmpty(name), "Resource groups configuration %s does not contain '%s'", configFile, NAME_PROPERTY);
+        checkState(!isNullOrEmpty(name), "Resource groups configuration %s does not contain '%s'", resolvedConfigurationPath(CONFIG_FILE), NAME_PROPERTY);
 
         setConfigurationManager(name, properties);
     }

diff --git a/core/trino-main/src/main/java/io/trino/security/AccessControlManager.java b/core/trino-main/src/main/java/io/trino/security/AccessControlManager.java
@@ -66,6 +66,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.nio.file.Path;
 import java.security.Principal;
 import java.util.Arrays;
 import java.util.Collection;
@@ -86,7 +87,9 @@
 import static com.google.common.base.Strings.isNullOrEmpty;
 import static com.google.common.base.Throwables.throwIfUnchecked;
 import static com.google.common.collect.ImmutableList.toImmutableList;
-import static io.airlift.configuration.ConfigurationLoader.loadPropertiesFrom;
+import static io.trino.plugin.base.ConfigurationLoader.configurationExists;
+import static io.trino.plugin.base.ConfigurationLoader.loadConfigurationFrom;
+import static io.trino.plugin.base.ConfigurationLoader.resolvedConfigurationPath;
 import static io.trino.spi.StandardErrorCode.INVALID_COLUMN_MASK;
 import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED;
 import static io.trino.spi.StandardErrorCode.SERVER_STARTING_UP;
@@ -99,7 +102,7 @@ public class AccessControlManager
 {
     private static final Logger log = Logger.get(AccessControlManager.class);
 
-    private static final File CONFIG_FILE = new File("etc/access-control.properties");
+    private static final Path CONFIG_FILE = Path.of("etc/access-control");
     private static final String NAME_PROPERTY = "access-control.name";
 
     private final NodeVersion nodeVersion;
@@ -163,12 +166,12 @@ public void loadSystemAccessControl()
     {
         List<File> configFiles = this.configFiles;
         if (configFiles.isEmpty()) {
-            if (!CONFIG_FILE.exists()) {
+            if (!configurationExists(CONFIG_FILE)) {
                 loadSystemAccessControl(defaultAccessControlName, ImmutableMap.of());
                 log.info("Using system access control: %s", defaultAccessControlName);
                 return;
             }
-            configFiles = ImmutableList.of(CONFIG_FILE);
+            configFiles = ImmutableList.of(resolvedConfigurationPath(CONFIG_FILE).toFile());
         }
 
         List<SystemAccessControl> systemAccessControls = configFiles.stream()
@@ -204,7 +207,7 @@ private SystemAccessControl createSystemAccessControl(File configFile)
 
         Map<String, String> properties;
         try {
-            properties = new HashMap<>(loadPropertiesFrom(configFile.getPath()));
+            properties = new HashMap<>(loadConfigurationFrom(configFile));
         }
         catch (IOException e) {
             throw new UncheckedIOException("Failed to read configuration file: " + configFile, e);

diff --git a/core/trino-main/src/main/java/io/trino/security/GroupProviderManager.java b/core/trino-main/src/main/java/io/trino/security/GroupProviderManager.java
@@ -23,8 +23,8 @@
 import io.trino.spi.security.GroupProvider;
 import io.trino.spi.security.GroupProviderFactory;
 
-import java.io.File;
 import java.io.IOException;
+import java.nio.file.Path;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
@@ -35,15 +35,17 @@
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.base.Strings.isNullOrEmpty;
-import static io.airlift.configuration.ConfigurationLoader.loadPropertiesFrom;
+import static io.trino.plugin.base.ConfigurationLoader.configurationExists;
+import static io.trino.plugin.base.ConfigurationLoader.loadConfigurationFrom;
+import static io.trino.plugin.base.ConfigurationLoader.resolvedConfigurationPath;
 import static java.lang.String.format;
 import static java.util.Objects.requireNonNull;
 
 public class GroupProviderManager
         implements GroupProvider
 {
     private static final Logger log = Logger.get(GroupProviderManager.class);
-    private static final File GROUP_PROVIDER_CONFIGURATION = new File("etc/group-provider.properties");
+    private static final Path GROUP_PROVIDER_CONFIGURATION = Path.of("etc/group-provider");
     private static final String GROUP_PROVIDER_PROPERTY_NAME = "group-provider.name";
     private final Map<String, GroupProviderFactory> groupProviderFactories = new ConcurrentHashMap<>();
     private final AtomicReference<Optional<GroupProvider>> configuredGroupProvider = new AtomicReference<>(Optional.empty());
@@ -71,17 +73,17 @@ public void loadConfiguredGroupProvider()
     }
 
     @VisibleForTesting
-    void loadConfiguredGroupProvider(File groupProviderFile)
+    void loadConfiguredGroupProvider(Path groupProviderFile)
             throws IOException
     {
-        if (configuredGroupProvider.get().isPresent() || !groupProviderFile.exists()) {
+        if (configuredGroupProvider.get().isPresent() || !configurationExists(groupProviderFile)) {
             return;
         }
-        Map<String, String> properties = new HashMap<>(loadPropertiesFrom(groupProviderFile.getPath()));
+        Map<String, String> properties = new HashMap<>(loadConfigurationFrom(groupProviderFile));
 
         String groupProviderName = properties.remove(GROUP_PROVIDER_PROPERTY_NAME);
         checkArgument(!isNullOrEmpty(groupProviderName),
-                "Group provider configuration %s does not contain %s", groupProviderFile.getAbsoluteFile(), GROUP_PROVIDER_PROPERTY_NAME);
+                "Group provider configuration %s does not contain %s", resolvedConfigurationPath(groupProviderFile), GROUP_PROVIDER_PROPERTY_NAME);
 
         setConfiguredGroupProvider(groupProviderName, properties);
     }