Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for loading configuration from toml files #22983

Closed
wants to merge 1 commit into from
Closed

Add support for loading configuration from toml files #22983

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions core/trino-main/src/main/java/io/trino/connector/CatalogStoreManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@
import io.trino.spi.classloader.ThreadContextClassLoader;
import io.trino.spi.connector.ConnectorName;

import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
Expand All @@ -37,14 +37,16 @@
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.base.Strings.isNullOrEmpty;
import static io.airlift.configuration.ConfigurationLoader.loadPropertiesFrom;
import static io.trino.plugin.base.ConfigurationLoader.configurationExists;
import static io.trino.plugin.base.ConfigurationLoader.loadConfigurationFrom;
import static io.trino.plugin.base.ConfigurationLoader.resolvedConfigurationPath;
import static java.util.Objects.requireNonNull;

public class CatalogStoreManager
implements CatalogStore
{
private static final Logger log = Logger.get(CatalogStoreManager.class);
private static final File CATALOG_STORE_CONFIGURATION = new File("etc/catalog-store.properties");
private static final Path CATALOG_STORE_CONFIGURATION = Path.of("etc/catalog-store");
private static final String CATALOG_STORE_PROPERTY_NAME = "catalog-store.name";
private final Map<String, CatalogStoreFactory> catalogStoreFactories = new ConcurrentHashMap<>();
private final AtomicReference<Optional<CatalogStore>> configuredCatalogStore = new AtomicReference<>(Optional.empty());
Expand Down Expand Up @@ -72,17 +74,17 @@ public void loadConfiguredCatalogStore()
}

@VisibleForTesting
void loadConfiguredCatalogStore(File catalogStoreFile)
void loadConfiguredCatalogStore(Path catalogStoreFile)
throws IOException
{
if (configuredCatalogStore.get().isPresent() || !catalogStoreFile.exists()) {
if (configuredCatalogStore.get().isPresent() || !configurationExists(catalogStoreFile)) {
return;
}
Map<String, String> properties = new HashMap<>(loadPropertiesFrom(catalogStoreFile.getPath()));
Map<String, String> properties = new HashMap<>(loadConfigurationFrom(catalogStoreFile));

String catalogStoreName = properties.remove(CATALOG_STORE_PROPERTY_NAME);
checkArgument(!isNullOrEmpty(catalogStoreName),
"Catalog store configuration %s does not contain %s", catalogStoreFile.getAbsoluteFile(), CATALOG_STORE_PROPERTY_NAME);
"Catalog store configuration %s does not contain %s", resolvedConfigurationPath(catalogStoreFile), CATALOG_STORE_PROPERTY_NAME);

setConfiguredCatalogStore(catalogStoreName, properties);
}
Expand Down
10 changes: 6 additions & 4 deletions core/trino-main/src/main/java/io/trino/connector/FileCatalogStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import com.google.inject.Inject;
import io.airlift.log.Logger;
import io.trino.connector.system.GlobalSystemConnector;
import io.trino.plugin.base.ConfigurationLoader;
import io.trino.spi.TrinoException;
import io.trino.spi.catalog.CatalogName;
import io.trino.spi.catalog.CatalogProperties;
Expand Down Expand Up @@ -48,7 +49,8 @@
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.common.io.Files.getNameWithoutExtension;
import static io.airlift.configuration.ConfigurationLoader.loadPropertiesFrom;
import static io.trino.plugin.base.ConfigurationLoader.PROPERTIES_SUFFIX;
import static io.trino.plugin.base.ConfigurationLoader.loadConfigurationFrom;
import static io.trino.spi.StandardErrorCode.CATALOG_STORE_ERROR;
import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED;
import static io.trino.spi.connector.CatalogHandle.createRootCatalogHandle;
Expand Down Expand Up @@ -151,7 +153,7 @@ private void checkModifiable()

private File toFile(CatalogName catalogName)
{
return new File(catalogsDirectory, catalogName.toString() + ".properties");
return new File(catalogsDirectory, catalogName.toString() + PROPERTIES_SUFFIX);
}

private static List<File> listCatalogFiles(File catalogsDirectory)
Expand All @@ -166,7 +168,7 @@ private static List<File> listCatalogFiles(File catalogsDirectory)
}
return Arrays.stream(files)
.filter(File::isFile)
.filter(file -> file.getName().endsWith(".properties"))
.filter(ConfigurationLoader::isConfigurationFile)
.collect(toImmutableList());
}

Expand Down Expand Up @@ -217,7 +219,7 @@ public CatalogProperties loadProperties()
{
Map<String, String> properties;
try {
properties = new HashMap<>(loadPropertiesFrom(file.getPath()));
properties = new HashMap<>(loadConfigurationFrom(file));
}
catch (IOException e) {
throw new UncheckedIOException("Error reading catalog property file " + file, e);
Expand Down
7 changes: 4 additions & 3 deletions core/trino-main/src/main/java/io/trino/connector/StaticCatalogManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import io.trino.connector.system.GlobalSystemConnector;
import io.trino.metadata.Catalog;
import io.trino.metadata.CatalogManager;
import io.trino.plugin.base.ConfigurationLoader;
import io.trino.server.ForStartup;
import io.trino.spi.TrinoException;
import io.trino.spi.catalog.CatalogName;
Expand Down Expand Up @@ -52,7 +53,7 @@
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static io.airlift.configuration.ConfigurationLoader.loadPropertiesFrom;
import static io.trino.plugin.base.ConfigurationLoader.loadConfigurationFrom;
import static io.trino.spi.StandardErrorCode.CATALOG_NOT_AVAILABLE;
import static io.trino.spi.StandardErrorCode.CATALOG_NOT_FOUND;
import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED;
Expand Down Expand Up @@ -93,7 +94,7 @@ public StaticCatalogManager(CatalogFactory catalogFactory, StaticCatalogManagerC

Map<String, String> properties;
try {
properties = new HashMap<>(loadPropertiesFrom(file.getPath()));
properties = new HashMap<>(loadConfigurationFrom(file));
}
catch (IOException e) {
throw new UncheckedIOException("Error reading catalog property file " + file, e);
Expand Down Expand Up @@ -128,7 +129,7 @@ private static List<File> listCatalogFiles(File catalogsDirectory)
}
return Arrays.stream(files)
.filter(File::isFile)
.filter(file -> file.getName().endsWith(".properties"))
.filter(ConfigurationLoader::isConfigurationFile)
.collect(toImmutableList());
}

Expand Down
13 changes: 8 additions & 5 deletions core/trino-main/src/main/java/io/trino/eventlistener/EventListenerManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
Expand All @@ -47,15 +48,17 @@
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.base.Strings.isNullOrEmpty;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static io.airlift.configuration.ConfigurationLoader.loadPropertiesFrom;
import static io.trino.plugin.base.ConfigurationLoader.configurationExists;
import static io.trino.plugin.base.ConfigurationLoader.loadConfigurationFrom;
import static io.trino.plugin.base.ConfigurationLoader.resolvedConfigurationPath;
import static java.lang.String.format;
import static java.util.Objects.requireNonNull;
import static java.util.concurrent.TimeUnit.MILLISECONDS;

public class EventListenerManager
{
private static final Logger log = Logger.get(EventListenerManager.class);
private static final File CONFIG_FILE = new File("etc/event-listener.properties");
private static final Path CONFIG_FILE = Path.of("etc/event-listener");
private static final String EVENT_LISTENER_NAME_PROPERTY = "event-listener.name";
private final List<File> configFiles;
private final int maxConcurrentQueryCompletedEvents;
Expand Down Expand Up @@ -109,10 +112,10 @@ private List<EventListener> configuredEventListeners()
{
List<File> configFiles = this.configFiles;
if (configFiles.isEmpty()) {
if (!CONFIG_FILE.exists()) {
if (!configurationExists(CONFIG_FILE)) {
return ImmutableList.of();
}
configFiles = ImmutableList.of(CONFIG_FILE);
configFiles = ImmutableList.of(resolvedConfigurationPath(CONFIG_FILE).toFile());
}
return configFiles.stream()
.map(this::createEventListener)
Expand Down Expand Up @@ -143,7 +146,7 @@ private EventListener createEventListener(File configFile)
private static Map<String, String> loadEventListenerProperties(File configFile)
{
try {
return new HashMap<>(loadPropertiesFrom(configFile.getPath()));
return new HashMap<>(loadConfigurationFrom(configFile.getPath()));
}
catch (IOException e) {
throw new UncheckedIOException("Failed to read configuration file: " + configFile, e);
Expand Down
13 changes: 7 additions & 6 deletions core/trino-main/src/main/java/io/trino/exchange/ExchangeManagerRegistry.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,17 +24,18 @@
import io.trino.spi.exchange.ExchangeManagerFactory;
import jakarta.annotation.PreDestroy;

import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.base.Strings.isNullOrEmpty;
import static io.airlift.configuration.ConfigurationLoader.loadPropertiesFrom;
import static io.trino.plugin.base.ConfigurationLoader.configurationExists;
import static io.trino.plugin.base.ConfigurationLoader.loadConfigurationFrom;
import static io.trino.spi.StandardErrorCode.EXCHANGE_MANAGER_NOT_CONFIGURED;
import static java.lang.String.format;
import static java.util.Objects.requireNonNull;
Expand All @@ -43,7 +44,7 @@ public class ExchangeManagerRegistry
{
private static final Logger log = Logger.get(ExchangeManagerRegistry.class);

private static final File CONFIG_FILE = new File("etc/exchange-manager.properties");
private static final Path CONFIG_FILE = Path.of("etc/exchange-manager");
private static final String EXCHANGE_MANAGER_NAME_PROPERTY = "exchange-manager.name";

private final OpenTelemetry openTelemetry;
Expand Down Expand Up @@ -74,7 +75,7 @@ public void addExchangeManagerFactory(ExchangeManagerFactory factory)

public void loadExchangeManager()
{
if (!CONFIG_FILE.exists()) {
if (!configurationExists(CONFIG_FILE)) {
return;
}

Expand Down Expand Up @@ -126,10 +127,10 @@ public void shutdown()
}
}

private static Map<String, String> loadProperties(File configFile)
private static Map<String, String> loadProperties(Path configFile)
{
try {
return new HashMap<>(loadPropertiesFrom(configFile.getPath()));
return new HashMap<>(loadConfigurationFrom(configFile));
}
catch (IOException e) {
throw new UncheckedIOException("Failed to read configuration file: " + configFile, e);
Expand Down
15 changes: 8 additions & 7 deletions ...no-main/src/main/java/io/trino/execution/resourcegroups/InternalResourceGroupManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
import org.weakref.jmx.MBeanExporter;
import org.weakref.jmx.Managed;

import java.io.File;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
Expand All @@ -55,7 +55,9 @@
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.base.Strings.isNullOrEmpty;
import static io.airlift.concurrent.Threads.daemonThreadsNamed;
import static io.airlift.configuration.ConfigurationLoader.loadPropertiesFrom;
import static io.trino.plugin.base.ConfigurationLoader.configurationExists;
import static io.trino.plugin.base.ConfigurationLoader.loadConfigurationFrom;
import static io.trino.plugin.base.ConfigurationLoader.resolvedConfigurationPath;
import static io.trino.spi.StandardErrorCode.QUERY_REJECTED;
import static java.lang.String.format;
import static java.util.Objects.requireNonNull;
Expand All @@ -68,7 +70,7 @@ public final class InternalResourceGroupManager<C>
{
private static final Logger log = Logger.get(InternalResourceGroupManager.class);

private static final File CONFIG_FILE = new File("etc/resource-groups.properties");
private static final Path CONFIG_FILE = Path.of("etc/resource-groups");
private static final String NAME_PROPERTY = "resource-groups.configuration-manager";

private final ScheduledExecutorService refreshExecutor = newSingleThreadScheduledExecutor(daemonThreadsNamed("ResourceGroupManager"));
Expand Down Expand Up @@ -141,15 +143,14 @@ public void addConfigurationManagerFactory(ResourceGroupConfigurationManagerFact
public void loadConfigurationManager()
throws Exception
{
File configFile = CONFIG_FILE.getAbsoluteFile();
if (!configFile.exists()) {
if (!configurationExists(CONFIG_FILE)) {
return;
}

Map<String, String> properties = new HashMap<>(loadPropertiesFrom(configFile.getPath()));
Map<String, String> properties = new HashMap<>(loadConfigurationFrom(CONFIG_FILE));

String name = properties.remove(NAME_PROPERTY);
checkState(!isNullOrEmpty(name), "Resource groups configuration %s does not contain '%s'", configFile, NAME_PROPERTY);
checkState(!isNullOrEmpty(name), "Resource groups configuration %s does not contain '%s'", resolvedConfigurationPath(CONFIG_FILE), NAME_PROPERTY);

setConfigurationManager(name, properties);
}
Expand Down
13 changes: 8 additions & 5 deletions core/trino-main/src/main/java/io/trino/security/AccessControlManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@
import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Path;
import java.security.Principal;
import java.util.Arrays;
import java.util.Collection;
Expand All @@ -86,7 +87,9 @@
import static com.google.common.base.Strings.isNullOrEmpty;
import static com.google.common.base.Throwables.throwIfUnchecked;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static io.airlift.configuration.ConfigurationLoader.loadPropertiesFrom;
import static io.trino.plugin.base.ConfigurationLoader.configurationExists;
import static io.trino.plugin.base.ConfigurationLoader.loadConfigurationFrom;
import static io.trino.plugin.base.ConfigurationLoader.resolvedConfigurationPath;
import static io.trino.spi.StandardErrorCode.INVALID_COLUMN_MASK;
import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED;
import static io.trino.spi.StandardErrorCode.SERVER_STARTING_UP;
Expand All @@ -99,7 +102,7 @@ public class AccessControlManager
{
private static final Logger log = Logger.get(AccessControlManager.class);

private static final File CONFIG_FILE = new File("etc/access-control.properties");
private static final Path CONFIG_FILE = Path.of("etc/access-control");
private static final String NAME_PROPERTY = "access-control.name";

private final NodeVersion nodeVersion;
Expand Down Expand Up @@ -163,12 +166,12 @@ public void loadSystemAccessControl()
{
List<File> configFiles = this.configFiles;
if (configFiles.isEmpty()) {
if (!CONFIG_FILE.exists()) {
if (!configurationExists(CONFIG_FILE)) {
loadSystemAccessControl(defaultAccessControlName, ImmutableMap.of());
log.info("Using system access control: %s", defaultAccessControlName);
return;
}
configFiles = ImmutableList.of(CONFIG_FILE);
configFiles = ImmutableList.of(resolvedConfigurationPath(CONFIG_FILE).toFile());
}

List<SystemAccessControl> systemAccessControls = configFiles.stream()
Expand Down Expand Up @@ -204,7 +207,7 @@ private SystemAccessControl createSystemAccessControl(File configFile)

Map<String, String> properties;
try {
properties = new HashMap<>(loadPropertiesFrom(configFile.getPath()));
properties = new HashMap<>(loadConfigurationFrom(configFile));
}
catch (IOException e) {
throw new UncheckedIOException("Failed to read configuration file: " + configFile, e);
Expand Down
16 changes: 9 additions & 7 deletions core/trino-main/src/main/java/io/trino/security/GroupProviderManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@
import io.trino.spi.security.GroupProvider;
import io.trino.spi.security.GroupProviderFactory;

import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
Expand All @@ -35,15 +35,17 @@
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.base.Strings.isNullOrEmpty;
import static io.airlift.configuration.ConfigurationLoader.loadPropertiesFrom;
import static io.trino.plugin.base.ConfigurationLoader.configurationExists;
import static io.trino.plugin.base.ConfigurationLoader.loadConfigurationFrom;
import static io.trino.plugin.base.ConfigurationLoader.resolvedConfigurationPath;
import static java.lang.String.format;
import static java.util.Objects.requireNonNull;

public class GroupProviderManager
implements GroupProvider
{
private static final Logger log = Logger.get(GroupProviderManager.class);
private static final File GROUP_PROVIDER_CONFIGURATION = new File("etc/group-provider.properties");
private static final Path GROUP_PROVIDER_CONFIGURATION = Path.of("etc/group-provider");
private static final String GROUP_PROVIDER_PROPERTY_NAME = "group-provider.name";
private final Map<String, GroupProviderFactory> groupProviderFactories = new ConcurrentHashMap<>();
private final AtomicReference<Optional<GroupProvider>> configuredGroupProvider = new AtomicReference<>(Optional.empty());
Expand Down Expand Up @@ -71,17 +73,17 @@ public void loadConfiguredGroupProvider()
}

@VisibleForTesting
void loadConfiguredGroupProvider(File groupProviderFile)
void loadConfiguredGroupProvider(Path groupProviderFile)
throws IOException
{
if (configuredGroupProvider.get().isPresent() || !groupProviderFile.exists()) {
if (configuredGroupProvider.get().isPresent() || !configurationExists(groupProviderFile)) {
return;
}
Map<String, String> properties = new HashMap<>(loadPropertiesFrom(groupProviderFile.getPath()));
Map<String, String> properties = new HashMap<>(loadConfigurationFrom(groupProviderFile));

String groupProviderName = properties.remove(GROUP_PROVIDER_PROPERTY_NAME);
checkArgument(!isNullOrEmpty(groupProviderName),
"Group provider configuration %s does not contain %s", groupProviderFile.getAbsoluteFile(), GROUP_PROVIDER_PROPERTY_NAME);
"Group provider configuration %s does not contain %s", resolvedConfigurationPath(groupProviderFile), GROUP_PROVIDER_PROPERTY_NAME);

setConfiguredGroupProvider(groupProviderName, properties);
}
Expand Down
Loading