Skip to content

Commit

Permalink
Allow list for JDBC properties
Browse files Browse the repository at this point in the history
  • Loading branch information
jihoonson committed Mar 23, 2021
1 parent 287c9be commit 48953e3
Show file tree
Hide file tree
Showing 33 changed files with 2,209 additions and 69 deletions.
63 changes: 63 additions & 0 deletions core/src/main/java/org/apache/druid/utils/ConnectionUriUtils.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.druid.utils;

import com.google.common.base.Preconditions;

import java.util.Set;

public final class ConnectionUriUtils
{
// Note: MySQL JDBC connector 8 supports 7 other protocols than just `jdbc:mysql:`
// (https://dev.mysql.com/doc/connector-j/8.0/en/connector-j-reference-jdbc-url-format.html).
// We should consider either expanding recognized mysql protocols or restricting allowed protocols to
// just a basic one.
public static final String MYSQL_PREFIX = "jdbc:mysql:";
public static final String POSTGRES_PREFIX = "jdbc:postgresql:";

/**
* This method checks {@param actualProperties} against {@param allowedProperties} if they are not system properties.
* A property is regarded as a system property if its name starts with a prefix in {@param systemPropertyPrefixes}.
* See org.apache.druid.server.initialization.JDBCAccessSecurityConfig for more details.
*
* If a non-system property that is not allowed is found, this method throws an {@link IllegalArgumentException}.
*/
public static void throwIfPropertiesAreNotAllowed(
Set<String> actualProperties,
Set<String> systemPropertyPrefixes,
Set<String> allowedProperties
)
{
for (String property : actualProperties) {
if (systemPropertyPrefixes.stream().noneMatch(property::startsWith)) {
Preconditions.checkArgument(
allowedProperties.contains(property),
"The property [%s] is not in the allowed list %s",
property,
allowedProperties
);
}
}
}

private ConnectionUriUtils()
{
}
}
40 changes: 40 additions & 0 deletions core/src/main/java/org/apache/druid/utils/Throwables.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.druid.utils;

public final class Throwables
{
public static boolean isThrowable(Throwable t, Class<? extends Throwable> searchFor)
{
if (t.getClass().isAssignableFrom(searchFor)) {
return true;
} else {
if (t.getCause() != null) {
return isThrowable(t.getCause(), searchFor);
} else {
return false;
}
}
}

private Throwables()
{
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.druid.utils;

import com.google.common.collect.ImmutableSet;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.runners.Enclosed;
import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;

@RunWith(Enclosed.class)
public class ConnectionUriUtilsTest
{
public static class ThrowIfURLHasNotAllowedPropertiesTest
{
@Rule
public ExpectedException expectedException = ExpectedException.none();

@Test
public void testEmptyActualProperties()
{
ConnectionUriUtils.throwIfPropertiesAreNotAllowed(
ImmutableSet.of(),
ImmutableSet.of("valid_key1", "valid_key2"),
ImmutableSet.of("system_key1", "system_key2")
);
}

@Test
public void testThrowForNonAllowedProperties()
{
expectedException.expect(IllegalArgumentException.class);
expectedException.expectMessage("The property [invalid_key] is not in the allowed list [valid_key1, valid_key2]");

ConnectionUriUtils.throwIfPropertiesAreNotAllowed(
ImmutableSet.of("valid_key1", "invalid_key"),
ImmutableSet.of("system_key1", "system_key2"),
ImmutableSet.of("valid_key1", "valid_key2")
);
}

@Test
public void testAllowedProperties()
{
ConnectionUriUtils.throwIfPropertiesAreNotAllowed(
ImmutableSet.of("valid_key2"),
ImmutableSet.of("system_key1", "system_key2"),
ImmutableSet.of("valid_key1", "valid_key2")
);
}

@Test
public void testAllowSystemProperties()
{
ConnectionUriUtils.throwIfPropertiesAreNotAllowed(
ImmutableSet.of("system_key1", "valid_key2"),
ImmutableSet.of("system_key1", "system_key2"),
ImmutableSet.of("valid_key1", "valid_key2")
);
}

@Test
public void testMatchSystemProperties()
{
ConnectionUriUtils.throwIfPropertiesAreNotAllowed(
ImmutableSet.of("system_key1.1", "system_key1.5", "system_key11.11", "valid_key2"),
ImmutableSet.of("system_key1", "system_key2"),
ImmutableSet.of("valid_key1", "valid_key2")
);
}
}
}
48 changes: 48 additions & 0 deletions core/src/test/java/org/apache/druid/utils/ThrowablesTest.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.druid.utils;

import org.junit.Assert;
import org.junit.Test;

public class ThrowablesTest
{
@Test
public void testIsThrowableItself()
{
Assert.assertTrue(Throwables.isThrowable(new NoClassDefFoundError(), NoClassDefFoundError.class));
}

@Test
public void testIsThrowableNestedThrowable()
{
Assert.assertTrue(
Throwables.isThrowable(new RuntimeException(new NoClassDefFoundError()), NoClassDefFoundError.class)
);
}

@Test
public void testIsThrowableNonTarget()
{
Assert.assertFalse(
Throwables.isThrowable(new RuntimeException(new ClassNotFoundException()), NoClassDefFoundError.class)
);
}
}
19 changes: 19 additions & 0 deletions docs/configuration/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -515,6 +515,25 @@ This deep storage is used to interface with Cassandra. Note that the `druid-cas
|`druid.storage.keyspace`|Cassandra key space.|none|


### Ingestion Security Configuration

#### JDBC Connections to External Databases

You can use the following properties to specify permissible JDBC options for:
- [SQL input source](../ingestion/native-batch.md#sql-input-source)
- [SQL firehose](../ingestion/native-batch.md#sqlfirehose),
- [globally cached JDBC lookups](../development/extensions-core/lookups-cached-global.md#jdbc-lookup)
- [JDBC Data Fetcher for per-lookup caching](../development/extensions-core/druid-lookups.md#data-fetcher-layer).

These properties do not apply to metadata storage connections.

|Property|Possible Values|Description|Default|
|--------|---------------|-----------|-------|
|`druid.access.jdbc.enforceAllowedProperties`|Boolean|When true, Druid applies `druid.access.jdbc.allowedProperties` to JDBC connections starting with `jdbc:postgresql:` or `jdbc:mysql:`. When false, Druid allows any kind of JDBC connections without JDBC property validation. This config is deprecated and will be removed in a future release.|false|
|`druid.access.jdbc.allowedProperties`|List of JDBC properties|Defines a list of allowed JDBC properties. Druid always enforces the list for all JDBC connections starting with `jdbc:postgresql:` or `jdbc:mysql:` if `druid.access.jdbc.enforceAllowedProperties` is set to true.<br/><br/>This option is tested against MySQL connector 5.1.48 and PostgreSQL connector 42.2.14. Other connector versions might not work.|["useSSL", "requireSSL", "ssl", "sslmode"]|
|`druid.access.jdbc.allowUnknownJdbcUrlFormat`|Boolean|When false, Druid only accepts JDBC connections starting with `jdbc:postgresql:` or `jdbc:mysql:`. When true, Druid allows JDBC connections to any kind of database, but only enforces `druid.access.jdbc.allowedProperties` for PostgreSQL and MySQL.|true|


### Task Logging

If you are running the indexing service in remote mode, the task logs must be stored in S3, Azure Blob Store, Google Cloud Storage or HDFS.
Expand Down
16 changes: 13 additions & 3 deletions docs/development/extensions-core/druid-lookups.md
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ Same for Loading cache, developer can implement a new type of loading cache by i

|Field|Type|Description|Required|default|
|-----|----|-----------|--------|-------|
|dataFetcher|JSON object|Specifies the lookup data fetcher type to use in order to fetch data|yes|null|
|dataFetcher|JSON object|Specifies the lookup data fetcher type for fetching data|yes|null|
|cacheFactory|JSON Object|Cache factory implementation|no |onHeapPolling|
|pollPeriod|Period|polling period |no |null (poll once)|

Expand Down Expand Up @@ -129,7 +129,7 @@ Guava cache configuration spec.
"type":"loadingLookup",
"dataFetcher":{ "type":"jdbcDataFetcher", "connectorConfig":"jdbc://mysql://localhost:3306/my_data_base", "table":"lookup_table_name", "keyColumn":"key_column_name", "valueColumn": "value_column_name"},
"loadingCacheSpec":{"type":"guava"},
"reverseLoadingCacheSpec":{"type":"guava", "maximumSize":500000, "expireAfterAccess":100000, "expireAfterAccess":10000}
"reverseLoadingCacheSpec":{"type":"guava", "maximumSize":500000, "expireAfterAccess":100000, "expireAfterWrite":10000}
}
```

Expand All @@ -150,6 +150,16 @@ Off heap cache is backed by [MapDB](http://www.mapdb.org/) implementation. MapDB
"type":"loadingLookup",
"dataFetcher":{ "type":"jdbcDataFetcher", "connectorConfig":"jdbc://mysql://localhost:3306/my_data_base", "table":"lookup_table_name", "keyColumn":"key_column_name", "valueColumn": "value_column_name"},
"loadingCacheSpec":{"type":"mapDb", "maxEntriesSize":100000},
"reverseLoadingCacheSpec":{"type":"mapDb", "maxStoreSize":5, "expireAfterAccess":100000, "expireAfterAccess":10000}
"reverseLoadingCacheSpec":{"type":"mapDb", "maxStoreSize":5, "expireAfterAccess":100000, "expireAfterWrite":10000}
}
```

### JDBC Data Fetcher

|Field|Type|Description|Required|default|
|-----|----|-----------|--------|-------|
|`connectorConfig`|JSON object|Specifies the database connection details. You can set `connectURI`, `user` and `password`. You can selectively allow JDBC properties in `connectURI`. See [JDBC connections security config](../../configuration/index.md#jdbc-connections-to-external-databases) for more details.|yes||
|`table`|string|The table name to read from.|yes||
|`keyColumn`|string|The column name that contains the lookup key.|yes||
|`valueColumn`|string|The column name that contains the lookup value.|yes||
|`streamingFetchSize`|int|Fetch size used in JDBC connections.|no|1000|
6 changes: 1 addition & 5 deletions docs/development/extensions-core/lookups-cached-global.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,6 @@ Globally cached lookups can be specified as part of the [cluster wide config for
"extractionNamespace": {
"type": "jdbc",
"connectorConfig": {
"createTables": true,
"connectURI": "jdbc:mysql:\/\/localhost:3306\/druid",
"user": "druid",
"password": "diurd"
Expand Down Expand Up @@ -107,7 +106,6 @@ In a simple case where only one [tier](../../querying/lookups.html#dynamic-confi
"extractionNamespace": {
"type": "jdbc",
"connectorConfig": {
"createTables": true,
"connectURI": "jdbc:mysql:\/\/localhost:3306\/druid",
"user": "druid",
"password": "diurd"
Expand Down Expand Up @@ -136,7 +134,6 @@ Where the Coordinator endpoint `/druid/coordinator/v1/lookups/realtime_customer2
"extractionNamespace": {
"type": "jdbc",
"connectorConfig": {
"createTables": true,
"connectURI": "jdbc:mysql://localhost:3306/druid",
"user": "druid",
"password": "diurd"
Expand Down Expand Up @@ -342,7 +339,7 @@ The JDBC lookups will poll a database to populate its local cache. If the `tsCol
|Parameter|Description|Required|Default|
|---------|-----------|--------|-------|
|`namespace`|The namespace to define|Yes||
|`connectorConfig`|The connector config to use|Yes||
|`connectorConfig`|The connector config to use. You can set `connectURI`, `user` and `password`. You can selectively allow JDBC properties in `connectURI`. See [JDBC connections security config](../../configuration/index.md#jdbc-connections-to-external-databases) for more details.|Yes||
|`table`|The table which contains the key value pairs|Yes||
|`keyColumn`|The column in `table` which contains the keys|Yes||
|`valueColumn`|The column in `table` which contains the values|Yes||
Expand All @@ -355,7 +352,6 @@ The JDBC lookups will poll a database to populate its local cache. If the `tsCol
"type":"jdbc",
"namespace":"some_lookup",
"connectorConfig":{
"createTables":true,
"connectURI":"jdbc:mysql://localhost:3306/druid",
"user":"druid",
"password":"diurd"
Expand Down
1 change: 0 additions & 1 deletion docs/development/extensions-core/mysql.md
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,6 @@ Copy or symlink this file to `extensions/mysql-metadata-storage` under the distr
|`druid.metadata.mysql.ssl.enabledSSLCipherSuites`|Overrides the existing cipher suites with these cipher suites.|none|no|
|`druid.metadata.mysql.ssl.enabledTLSProtocols`|Overrides the TLS protocols with these protocols.|none|no|


### MySQL Firehose

The MySQL extension provides an implementation of an [SqlFirehose](../../ingestion/native-batch.md#firehoses-deprecated) which can be used to ingest data into Druid from a MySQL database.
Expand Down
4 changes: 2 additions & 2 deletions docs/ingestion/native-batch.md
Original file line number Diff line number Diff line change
Expand Up @@ -1346,7 +1346,7 @@ Please refer to the Recommended practices section below before using this input
|property|description|required?|
|--------|-----------|---------|
|type|This should be "sql".|Yes|
|database|Specifies the database connection details. The database type corresponds to the extension that supplies the `connectorConfig` support and this extension must be loaded into Druid. For database types `mysql` and `postgresql`, the `connectorConfig` support is provided by [mysql-metadata-storage](../development/extensions-core/mysql.md) and [postgresql-metadata-storage](../development/extensions-core/postgresql.md) extensions respectively.|Yes|
|database|Specifies the database connection details. The database type corresponds to the extension that supplies the `connectorConfig` support. The specified extension must be loaded into Druid:<br/><br/><ul><li>[mysql-metadata-storage](../development/extensions-core/mysql.md) for `mysql`</li><li> [postgresql-metadata-storage](../development/extensions-core/postgresql.md) extension for `postgresql`.</li></ul><br/><br/>You can selectively allow JDBC properties in `connectURI`. See [JDBC connections security config](../configuration/index.md#jdbc-connections-to-external-databases) for more details.|Yes|
|foldCase|Toggle case folding of database column names. This may be enabled in cases where the database returns case insensitive column names in query results.|No|
|sqls|List of SQL queries where each SQL query would retrieve the data to be indexed.|Yes|

Expand Down Expand Up @@ -1692,7 +1692,7 @@ Requires one of the following extensions:
|property|description|default|required?|
|--------|-----------|-------|---------|
|type|This should be "sql".||Yes|
|database|Specifies the database connection details.||Yes|
|database|Specifies the database connection details. The database type corresponds to the extension that supplies the `connectorConfig` support. The specified extension must be loaded into Druid:<br/><br/><ul><li>[mysql-metadata-storage](../development/extensions-core/mysql.md) for `mysql`</li><li> [postgresql-metadata-storage](../development/extensions-core/postgresql.md) extension for `postgresql`.</li></ul><br/><br/>You can selectively allow JDBC properties in `connectURI`. See [JDBC connections security config](../configuration/index.md#jdbc-connections-to-external-databases) for more details.||Yes|
|maxCacheCapacityBytes|Maximum size of the cache space in bytes. 0 means disabling cache. Cached files are not removed until the ingestion task completes.|1073741824|No|
|maxFetchCapacityBytes|Maximum size of the fetch space in bytes. 0 means disabling prefetch. Prefetched files are removed immediately once they are read.|1073741824|No|
|prefetchTriggerBytes|Threshold to trigger prefetching SQL result objects.|maxFetchCapacityBytes / 2|No|
Expand Down
9 changes: 6 additions & 3 deletions extensions-core/lookups-cached-global/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -110,12 +110,15 @@
<artifactId>jsr311-api</artifactId>
<scope>provided</scope>
</dependency>

<!-- Included to improve the out-of-the-box experience for supported JDBC connectors -->
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>${mysql.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<scope>runtime</scope>
</dependency>

<!-- Tests -->
Expand Down
Loading

0 comments on commit 48953e3

Please sign in to comment.