Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add new option for rename response column names #923

Merged
merged 1 commit into from
May 7, 2022
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -123,6 +123,11 @@ public enum ClickHouseClientOption implements ClickHouseOption {
*/
MAX_THREADS_PER_CLIENT("max_threads_per_client", 0,
"Size of thread pool for each client instance, 0 or negative number means the client will use shared thread pool."),
/**
* Method to rename response columns.
*/
RENAME_RESPONSE_COLUMN("rename_response_column", ClickHouseRenameMethod.NONE,
"Method to rename response columns."),
/**
* Whether to enable retry.
*/
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
package com.clickhouse.client.config;

import java.util.function.UnaryOperator;

/**
* Methods for renaming.
*/
public enum ClickHouseRenameMethod {
/**
* No OP.
*/
NONE(null),
/**
* Removes prefix including the dot. So "d.t.col1" becomes "col1" and "col2"
* remains the same.
*/
REMOVE_PREFIX(s -> {
int index = s.lastIndexOf('.');
return index >= 0 ? s.substring(index + 1) : s;
}),

/**
* Replaces whitespace and underscore to camel case. So "a simple_column"
* becomes "aSimpleColumn" and "col_1 2" becomes "col12".
*/
TO_CAMELCASE(s -> {
StringBuilder builder = new StringBuilder(s.length());
boolean toUpperCase = false;
for (char ch : s.toCharArray()) {
if (Character.isWhitespace(ch) || ch == '_') {
toUpperCase = true;
} else if (toUpperCase) {
builder.append(Character.toUpperCase(ch));
toUpperCase = false;
} else {
builder.append(ch);
}
}
return builder.toString();
}),
/**
* Removes prefix and replace whitespace and underscore to camel case.
*/
TO_CAMELCASE_WITHOUT_PREFIX(s -> TO_CAMELCASE.rename(REMOVE_PREFIX.rename(s))),
/**
* Replaces whitespace and camel case to underscore. So "aSimpleColumn" becomes
* "a_simple_column" and "col12" becomes "col_12".
*/
TO_UNDERSCORE(s -> {
StringBuilder builder = new StringBuilder(s.length() + 5);
int prev = -1; // 0 - normal, 1 - whitespace, 2 - upper case
for (char ch : s.toCharArray()) {
if (Character.isWhitespace(ch)) {
if (prev == 0) {
builder.append('_');
}
prev = 1;
} else if (Character.isUpperCase(ch)) {
if (prev == 0) {
builder.append('_').append(Character.toLowerCase(ch));
} else if (prev == 1) {
builder.append(Character.toLowerCase(ch));
} else {
builder.append(ch);
}
prev = 2;
} else {
builder.append(ch);
prev = 0;
}
}
return builder.toString();
}),
/**
* Removes prefix and replace whitespace and camel case to underscore.
*/
TO_UNDERSCORE_WITHOUT_PREFIX(s -> TO_UNDERSCORE.rename(REMOVE_PREFIX.rename(s)));

private final UnaryOperator<String> renameFunc;

ClickHouseRenameMethod(UnaryOperator<String> renameFunc) {
this.renameFunc = renameFunc;
}

/**
* Rename the given name.
*
* @param name name to change
* @return non-null new name
*/
public String rename(String name) {
if (name == null) {
name = "";
}
return renameFunc != null ? renameFunc.apply(name) : name;
}
}
Original file line number Diff line number Diff line change
@@ -25,6 +25,8 @@
import com.clickhouse.client.ClickHouseSerializer;
import com.clickhouse.client.ClickHouseValue;
import com.clickhouse.client.ClickHouseValues;
import com.clickhouse.client.config.ClickHouseClientOption;
import com.clickhouse.client.config.ClickHouseRenameMethod;

/**
* Data processor for handling {@link ClickHouseFormat#RowBinary} and
@@ -521,10 +523,12 @@ protected List<ClickHouseColumn> readColumns() throws IOException {
names[i] = input.readUnicodeString();
}

ClickHouseRenameMethod m = (ClickHouseRenameMethod) config
.getOption(ClickHouseClientOption.RENAME_RESPONSE_COLUMN);
List<ClickHouseColumn> columns = new ArrayList<>(size);
for (int i = 0; i < size; i++) {
// a bit risky here - what if ClickHouse support user type?
columns.add(ClickHouseColumn.of(names[i], input.readAsciiString()));
columns.add(ClickHouseColumn.of(m.rename(names[i]), input.readAsciiString()));
}

return columns;
Original file line number Diff line number Diff line change
@@ -20,6 +20,8 @@
import com.clickhouse.client.ClickHouseRecord;
import com.clickhouse.client.ClickHouseSerializer;
import com.clickhouse.client.ClickHouseValue;
import com.clickhouse.client.config.ClickHouseClientOption;
import com.clickhouse.client.config.ClickHouseRenameMethod;
import com.clickhouse.client.data.tsv.ByteFragment;

public class ClickHouseTabSeparatedProcessor extends ClickHouseDataProcessor {
@@ -288,10 +290,12 @@ protected List<ClickHouseColumn> readColumns() throws IOException {
buf.lastByte() == getTextHandler().rowDelimiter ? buf.length() - 1 : buf.length());
types = toStringArray(typesFragment, getTextHandler().colDelimiter);
}
List<ClickHouseColumn> list = new ArrayList<>(cols.length);

ClickHouseRenameMethod m = (ClickHouseRenameMethod) config
.getOption(ClickHouseClientOption.RENAME_RESPONSE_COLUMN);
List<ClickHouseColumn> list = new ArrayList<>(cols.length);
for (int i = 0; i < cols.length; i++) {
list.add(ClickHouseColumn.of(cols[i], types == null ? "Nullable(String)" : types[i]));
list.add(ClickHouseColumn.of(m.rename(cols[i]), types == null ? "Nullable(String)" : types[i]));
}

return list;
Original file line number Diff line number Diff line change
@@ -23,6 +23,7 @@
import java.util.stream.Collectors;

import com.clickhouse.client.config.ClickHouseClientOption;
import com.clickhouse.client.config.ClickHouseRenameMethod;
import com.clickhouse.client.config.ClickHouseSslMode;
import com.clickhouse.client.data.BinaryStreamUtils;
import com.clickhouse.client.data.ClickHouseBigDecimalValue;
@@ -93,6 +94,18 @@ protected Object[][] getCompressionMatrix() {
new Object[] { false, true } };
}

@DataProvider(name = "renameMethods")
protected Object[][] getRenameMethods() {
return new Object[][] {
new Object[] { null, "a b c", " ", "d.E_f" },
new Object[] { ClickHouseRenameMethod.NONE, "a b c", " ", "d.E_f" },
new Object[] { ClickHouseRenameMethod.REMOVE_PREFIX, "a b c", " ", "E_f" },
new Object[] { ClickHouseRenameMethod.TO_CAMELCASE, "aBC", "", "d.EF" },
new Object[] { ClickHouseRenameMethod.TO_CAMELCASE_WITHOUT_PREFIX, "aBC", "", "EF" },
new Object[] { ClickHouseRenameMethod.TO_UNDERSCORE, "a_b_c", "", "d._e_f" },
new Object[] { ClickHouseRenameMethod.TO_UNDERSCORE_WITHOUT_PREFIX, "a_b_c", "", "E_f" }, };
}

@DataProvider(name = "simpleTypeProvider")
protected Object[][] getSimpleTypes() {
return new Object[][] {
@@ -1073,6 +1086,21 @@ public void testInsertWithInputFunction() throws Exception {
}
}

@Test(dataProvider = "renameMethods", groups = "integration")
public void testRenameResponseColumns(ClickHouseRenameMethod m, String col1, String col2, String col3)
throws Exception {
ClickHouseNode server = getServer();
try (ClickHouseClient client = getClient();
ClickHouseResponse resp = client.connect(server)
.format(ClickHouseFormat.RowBinaryWithNamesAndTypes)
.option(ClickHouseClientOption.RENAME_RESPONSE_COLUMN, m)
.query("select 1 `a b c`, 2 ` `, 3 `d.E_f`").execute().get()) {
Assert.assertEquals(resp.getColumns().get(0).getColumnName(), col1);
Assert.assertEquals(resp.getColumns().get(1).getColumnName(), col2);
Assert.assertEquals(resp.getColumns().get(2).getColumnName(), col3);
}
}

@Test(groups = "integration")
public void testTempTable() throws Exception {
ClickHouseNode server = getServer();
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
package com.clickhouse.client.config;

import org.testng.Assert;
import org.testng.annotations.Test;

public class ClickHouseRenameMethodTest {
@Test(groups = { "unit" })
public void testRenameNullOrEmptyString() {
for (ClickHouseRenameMethod m : ClickHouseRenameMethod.values()) {
Assert.assertEquals(m.rename(null), "");
Assert.assertEquals(m.rename(""), "");
}
}

@Test(groups = { "unit" })
public void testNone() {
Assert.assertEquals(ClickHouseRenameMethod.NONE.rename("\t \n \r"), "\t \n \r");
Assert.assertEquals(ClickHouseRenameMethod.NONE.rename("test 1 2 3"), "test 1 2 3");
}

@Test(groups = { "unit" })
public void testRemovePrefix() {
Assert.assertEquals(ClickHouseRenameMethod.REMOVE_PREFIX.rename("\t \n \r"), "\t \n \r");
Assert.assertEquals(ClickHouseRenameMethod.REMOVE_PREFIX.rename("test 1 2 3"), "test 1 2 3");
Assert.assertEquals(ClickHouseRenameMethod.REMOVE_PREFIX.rename("test.1 2 3"), "1 2 3");
Assert.assertEquals(ClickHouseRenameMethod.REMOVE_PREFIX.rename("test.1.2.3"), "3");
Assert.assertEquals(ClickHouseRenameMethod.REMOVE_PREFIX.rename(".test"), "test");
Assert.assertEquals(ClickHouseRenameMethod.REMOVE_PREFIX.rename("test."), "");
Assert.assertEquals(ClickHouseRenameMethod.REMOVE_PREFIX.rename("."), "");
}

@Test(groups = { "unit" })
public void testCamelCase() {
Assert.assertEquals(ClickHouseRenameMethod.TO_CAMELCASE.rename("\t \n \r"), "");
Assert.assertEquals(ClickHouseRenameMethod.TO_CAMELCASE.rename("test 1 2 3"), "test123");
Assert.assertEquals(ClickHouseRenameMethod.TO_CAMELCASE.rename("test oNE Two_three"), "testONETwoThree");
Assert.assertEquals(ClickHouseRenameMethod.TO_CAMELCASE.rename("test"), "test");
Assert.assertEquals(ClickHouseRenameMethod.TO_CAMELCASE.rename(" test"), "Test");
Assert.assertEquals(ClickHouseRenameMethod.TO_CAMELCASE.rename("test "), "test");

Assert.assertEquals(ClickHouseRenameMethod.TO_CAMELCASE_WITHOUT_PREFIX.rename("a.test_col"), "testCol");
}

@Test(groups = { "unit" })
public void testUnderscore() {
Assert.assertEquals(ClickHouseRenameMethod.TO_UNDERSCORE.rename("\t \n \r"), "");
Assert.assertEquals(ClickHouseRenameMethod.TO_UNDERSCORE.rename("TEST"), "TEST");
Assert.assertEquals(ClickHouseRenameMethod.TO_UNDERSCORE.rename("Test"), "Test");
Assert.assertEquals(ClickHouseRenameMethod.TO_UNDERSCORE.rename("TestONE"), "Test_oNE");
Assert.assertEquals(ClickHouseRenameMethod.TO_UNDERSCORE.rename("Test ONE"), "Test_oNE");
Assert.assertEquals(ClickHouseRenameMethod.TO_UNDERSCORE.rename("Test oneTwo"), "Test_one_two");
Assert.assertEquals(ClickHouseRenameMethod.TO_UNDERSCORE.rename("testOnetWo"), "test_onet_wo");
Assert.assertEquals(ClickHouseRenameMethod.TO_UNDERSCORE.rename("test12Three"), "test12_three");

Assert.assertEquals(ClickHouseRenameMethod.TO_UNDERSCORE_WITHOUT_PREFIX.rename("a.t.est1\t 2Three"),
"est1_2_three");
}
}