Skip to content

Commit

Permalink
Merge branch 'main' into v2_connection_metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
@Paultagoras
Paultagoras authored Feb 13, 2025
2 parents 264fa7a + 208d547 commit 20314c6
Show file tree
Hide file tree
Showing 49 changed files with 1,725 additions and 196 deletions.
9 changes: 8 additions & 1 deletion clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseRequest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import java.util.function.Function;

import com.clickhouse.client.config.ClickHouseClientOption;
import com.clickhouse.client.config.ClickHouseDefaults;
import com.clickhouse.config.ClickHouseConfigChangeListener;
import com.clickhouse.config.ClickHouseOption;
import com.clickhouse.data.ClickHouseChecker;
Expand Down Expand Up @@ -590,7 +591,13 @@ public ClickHouseConfig getConfig() {
Map<ClickHouseOption, Serializable> merged = new HashMap<>();
merged.putAll(clientConfig.getAllOptions());
merged.putAll(options);
config = new ClickHouseConfig(merged, node.getCredentials(clientConfig),

ClickHouseCredentials credentials = node.getCredentials(clientConfig);
if (merged.containsKey(ClickHouseDefaults.USER) && merged.containsKey(ClickHouseDefaults.PASSWORD)) {
credentials = ClickHouseCredentials.fromUserAndPassword((String) merged.get(ClickHouseDefaults.USER),
(String) merged.get(ClickHouseDefaults.PASSWORD));
}
config = new ClickHouseConfig(merged, credentials,
clientConfig.getNodeSelector(), clientConfig.getMetricRegistry().orElse(null));
}
}
Expand Down
6 changes: 4 additions & 2 deletions clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseServerForTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -250,7 +250,9 @@ public static ClickHouseNode getClickHouseNode(ClickHouseProtocol protocol, bool
}
}

return ClickHouseNode.builder(template).address(protocol, new InetSocketAddress(host, port)).build();
return ClickHouseNode.builder(template).address(protocol, new InetSocketAddress(host, port))
.credentials(new ClickHouseCredentials("default", getPassword()))
.build();
}

public static ClickHouseNode getClickHouseNode(ClickHouseProtocol protocol, int port) {
Expand Down Expand Up @@ -314,7 +316,7 @@ public static String getPassword() {
if (isCloud) {
return System.getenv("CLICKHOUSE_CLOUD_PASSWORD");
} else {
return "";
return "test_default_password";
}
}

Expand Down
3 changes: 2 additions & 1 deletion clickhouse-client/src/test/java/com/clickhouse/client/ClientIntegrationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2511,7 +2511,7 @@ public void testTransactionTimeout() throws ClickHouseException {

tx.begin();
try {
Thread.sleep(3000L);
Thread.sleep(4000L);
} catch (InterruptedException ex) {
Assert.fail("Sleep was interrupted", ex);
}
Expand Down Expand Up @@ -2706,6 +2706,7 @@ public void testFailover() throws ClickHouseException {
ClickHouseNode availableNode = getServer();
Properties props = new Properties();
props.setProperty("failover", "1");
props.setProperty(ClickHouseDefaults.PASSWORD.getKey(), getPassword());

// nodes with the first node is unavailable
ClickHouseNodes nodes = ClickHouseNodes.of(
Expand Down
13 changes: 11 additions & 2 deletions clickhouse-client/src/test/resources/containers/clickhouse-server/users.d/users.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,15 @@
</managed2>
</profiles>
<users>
<default>
<profile>default</profile>
<networks>
<ip>::/0</ip>
</networks>
<password>test_default_password</password>
<quota>default</quota>
<access_management>1</access_management>
</default>
<dba>
<access_management>1</access_management>
<profile>default</profile>
Expand Down Expand Up @@ -40,15 +49,15 @@
<networks>
<ip>::/0</ip>
</networks>
<password></password>
<password>poorman_111</password>
<quota>default</quota>
</poorman1>
<poorman2>
<profile>managed2</profile>
<networks>
<ip>::/0</ip>
</networks>
<password></password>
<password>poorman_222</password>
<quota>default</quota>
</poorman2>
<test>
Expand Down
110 changes: 109 additions & 1 deletion clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,19 @@

import java.io.Serializable;
import java.lang.reflect.Array;
import java.math.BigInteger;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.TimeZone;

/**
Expand All @@ -65,6 +71,7 @@ public final class ClickHouseColumn implements Serializable {
private static final String KEYWORD_OBJECT = ClickHouseDataType.Object.name();
private static final String KEYWORD_MAP = ClickHouseDataType.Map.name();
private static final String KEYWORD_NESTED = ClickHouseDataType.Nested.name();
private static final String KEYWORD_VARIANT = ClickHouseDataType.Variant.name();

private int columnCount;
private int columnIndex;
Expand Down Expand Up @@ -92,6 +99,14 @@ public final class ClickHouseColumn implements Serializable {

private ClickHouseValue template;

private Map<Class<?>, Integer> classToVariantOrdNumMap;

private Map<Class<?>, Integer> arrayToVariantOrdNumMap;

private Map<Class<?>, Integer> mapKeyToVariantOrdNumMap;
private Map<Class<?>, Integer> mapValueToVariantOrdNumMap;


private static ClickHouseColumn update(ClickHouseColumn column) {
column.enumConstants = ClickHouseEnum.EMPTY;
int size = column.parameters.size();
Expand Down Expand Up @@ -273,6 +288,9 @@ private static ClickHouseColumn update(ClickHouseColumn column) {
case Nothing:
column.template = ClickHouseEmptyValue.INSTANCE;
break;
case Variant:
column.template = ClickHouseTupleValue.of();
break;
default:
break;
}
Expand Down Expand Up @@ -398,7 +416,8 @@ protected static int readColumn(String args, int startIndex, int len, String nam
fixedLength = false;
estimatedLength++;
} else if (args.startsWith(matchedKeyword = KEYWORD_TUPLE, i)
|| args.startsWith(matchedKeyword = KEYWORD_OBJECT, i)) {
|| args.startsWith(matchedKeyword = KEYWORD_OBJECT, i)
|| args.startsWith(matchedKeyword = KEYWORD_VARIANT, i)) {
int index = args.indexOf('(', i + matchedKeyword.length());
if (index < i) {
throw new IllegalArgumentException(ERROR_MISSING_NESTED_TYPE);
Expand All @@ -410,12 +429,22 @@ protected static int readColumn(String args, int startIndex, int len, String nam
if (c == ')') {
break;
} else if (c != ',' && !Character.isWhitespace(c)) {
String columnName = "";
i = readColumn(args, i, endIndex, "", nestedColumns);
}
}
if (nestedColumns.isEmpty()) {
throw new IllegalArgumentException("Tuple should have at least one nested column");
}

List<ClickHouseDataType> variantDataTypes = new ArrayList<>();
if (matchedKeyword.equals(KEYWORD_VARIANT)) {
nestedColumns.sort(Comparator.comparing(o -> o.getDataType().name()));
nestedColumns.forEach(c -> {
c.columnName = "v." + c.getDataType().name();
variantDataTypes.add(c.dataType);
});
}
column = new ClickHouseColumn(ClickHouseDataType.valueOf(matchedKeyword), name,
args.substring(startIndex, endIndex + 1), nullable, lowCardinality, null, nestedColumns);
for (ClickHouseColumn n : nestedColumns) {
Expand All @@ -424,6 +453,39 @@ protected static int readColumn(String args, int startIndex, int len, String nam
fixedLength = false;
}
}
column.classToVariantOrdNumMap = ClickHouseDataType.buildVariantMapping(variantDataTypes);

for (int ordNum = 0; ordNum < nestedColumns.size(); ordNum++) {
ClickHouseColumn nestedColumn = nestedColumns.get(ordNum);
if (nestedColumn.getDataType() == ClickHouseDataType.Array) {
Set<Class<?>> classSet = ClickHouseDataType.DATA_TYPE_TO_CLASS.get(nestedColumn.arrayBaseColumn.dataType);
if (classSet != null) {
if (column.arrayToVariantOrdNumMap == null) {
column.arrayToVariantOrdNumMap = new HashMap<>();
}
for (Class<?> c : classSet) {
column.arrayToVariantOrdNumMap.put(c, ordNum);
}
}
} else if (nestedColumn.getDataType() == ClickHouseDataType.Map) {
Set<Class<?>> keyClassSet = ClickHouseDataType.DATA_TYPE_TO_CLASS.get(nestedColumn.getKeyInfo().getDataType());
Set<Class<?>> valueClassSet = ClickHouseDataType.DATA_TYPE_TO_CLASS.get(nestedColumn.getValueInfo().getDataType());
if (keyClassSet != null && valueClassSet != null) {
if (column.mapKeyToVariantOrdNumMap == null) {
column.mapKeyToVariantOrdNumMap = new HashMap<>();
}
if (column.mapValueToVariantOrdNumMap == null) {
column.mapValueToVariantOrdNumMap = new HashMap<>();
}
for (Class<?> c : keyClassSet) {
column.mapKeyToVariantOrdNumMap.put(c, ordNum);
}
for (Class<?> c : valueClassSet) {
column.mapValueToVariantOrdNumMap.put(c, ordNum);
}
}
}
}
}

if (column == null) {
Expand Down Expand Up @@ -627,6 +689,52 @@ public boolean isAggregateFunction() {

}

public int getVariantOrdNum(Object value) {
if (value != null && value.getClass().isArray()) {
// TODO: add cache by value class
Class<?> c = value.getClass();
while (c.isArray()) {
c = c.getComponentType();
}
return arrayToVariantOrdNumMap.getOrDefault(c, -1);
} else if (value != null && value instanceof List<?>) {
// TODO: add cache by instance of the list
Object tmpV = ((List) value).get(0);
Class<?> valueClass = tmpV.getClass();
while (tmpV instanceof List<?>) {
tmpV = ((List) tmpV).get(0);
valueClass = tmpV.getClass();
}
return arrayToVariantOrdNumMap.getOrDefault(valueClass, -1);
} else if (value != null && value instanceof Map<?,?>) {
// TODO: add cache by instance of map
Map<?, ?> map = (Map<?, ?>) value;
if (!map.isEmpty()) {
for (Map.Entry<?, ?> e : map.entrySet()) {
if (e.getValue() != null) {
int keyOrdNum = mapKeyToVariantOrdNumMap.getOrDefault(e.getKey().getClass(), -1);
int valueOrdNum = mapValueToVariantOrdNumMap.getOrDefault(e.getValue().getClass(), -1);

if (keyOrdNum == valueOrdNum) {
return valueOrdNum; // exact match
} else if (keyOrdNum != -1 && valueOrdNum != -1) {
if (ClickHouseDataType.DATA_TYPE_TO_CLASS.get(nested.get(keyOrdNum).getValueInfo().getDataType()).contains(e.getValue().getClass())){
return keyOrdNum; // can write to map found by key class because values are compatible
} else {
return valueOrdNum;
}
}

break;
}
}
}
return -1;
} else {
return classToVariantOrdNumMap.getOrDefault(value.getClass(), -1);
}
}

public boolean isArray() {
return dataType == ClickHouseDataType.Array;
}
Expand Down
Loading

0 comments on commit 20314c6

Please sign in to comment.