From cf361915d5a70ce3f05f7be29c6686465c6ca62e Mon Sep 17 00:00:00 2001 From: iNikitaGricenko Date: Fri, 13 Oct 2023 15:30:03 +0300 Subject: [PATCH 1/9] add dremio plugin --- build.gradle | 1 + docker-compose-ci.yml | 7 ++ plugin-jdbc-dremio/build.gradle | 21 +++++ .../jdbc/dremio/DremioCellConverter.java | 86 +++++++++++++++++++ .../io/kestra/plugin/jdbc/dremio/Query.java | 60 +++++++++++++ .../io/kestra/plugin/jdbc/dremio/Trigger.java | 76 ++++++++++++++++ .../plugin/jdbc/dremio/package-info.java | 7 ++ .../icons/io.kestra.plugin.jdbc.dremio.svg | 54 ++++++++++++ .../src/main/resources/icons/plugin-icon.svg | 54 ++++++++++++ settings.gradle | 1 + 10 files changed, 367 insertions(+) create mode 100644 plugin-jdbc-dremio/build.gradle create mode 100644 plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/DremioCellConverter.java create mode 100644 plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Query.java create mode 100644 plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Trigger.java create mode 100644 plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/package-info.java create mode 100644 plugin-jdbc-dremio/src/main/resources/icons/io.kestra.plugin.jdbc.dremio.svg create mode 100644 plugin-jdbc-dremio/src/main/resources/icons/plugin-icon.svg diff --git a/build.gradle b/build.gradle index c758b771..0dbec984 100644 --- a/build.gradle +++ b/build.gradle @@ -29,6 +29,7 @@ subprojects { mavenCentral() if (isBuildSnapshot) { maven { url "https://s01.oss.sonatype.org/content/repositories/snapshots/" } + maven { url "https://maven.dremio.com/free/" } } } diff --git a/docker-compose-ci.yml b/docker-compose-ci.yml index 66d2ad9c..a16bf2a5 100644 --- a/docker-compose-ci.yml +++ b/docker-compose-ci.yml @@ -67,3 +67,10 @@ services: - QuickStart - -type - batch + + dremio: + image: dremio/dremio-oss + ports: + - "9047:9047" + - "31010:31010" + - "45678:45678" \ No newline at end of file diff --git a/plugin-jdbc-dremio/build.gradle b/plugin-jdbc-dremio/build.gradle new file mode 100644 index 00000000..6d9a8809 --- /dev/null +++ b/plugin-jdbc-dremio/build.gradle @@ -0,0 +1,21 @@ +project.description = 'Query Dremio databases using the Kestra JDBC plugin.' + +jar { + manifest { + attributes( + "X-Kestra-Name": project.name, + "X-Kestra-Title": "Dremio", + "X-Kestra-Group": project.group + ".jdbc.dremio", + "X-Kestra-Description": project.description, + "X-Kestra-Version": project.version + ) + } +} + +dependencies { + implementation("com.dremio.distribution:dremio-jdbc-driver:3.0.6-201812082352540436-1f684f9") + implementation("org.apache.arrow:flight-sql-jdbc-driver:12.0.1") + implementation project(':plugin-jdbc') + + testImplementation project(':plugin-jdbc').sourceSets.test.output +} \ No newline at end of file diff --git a/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/DremioCellConverter.java b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/DremioCellConverter.java new file mode 100644 index 00000000..0309b626 --- /dev/null +++ b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/DremioCellConverter.java @@ -0,0 +1,86 @@ +package io.kestra.plugin.jdbc.dremio; + +import io.kestra.plugin.jdbc.AbstractCellConverter; + +import java.net.Inet4Address; +import java.net.Inet6Address; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class DremioCellConverter extends AbstractCellConverter { + private static final Pattern PATTERN = Pattern.compile("DateTime(64)?\\((.*)'(.*)'\\)"); + private static final DateTimeFormatter DATE_TIME_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss[.SSSSSS][.SSSSS][.SSSS][.SSS][.SS][.S]"); + + public DremioCellConverter(ZoneId zoneId) { + super(zoneId); + } + + @Override + public Object convertCell(int columnIndex, ResultSet resultSet, Connection connection) throws SQLException { + Object data = resultSet.getObject(columnIndex); + + if (data == null) { + return null; + } + + Object columnValue = resultSet.getObject(columnIndex); + String columnTypeName = resultSet.getMetaData().getColumnTypeName(columnIndex); + + if (columnTypeName.equals("DateTime")) { + // date with no TZ, we use the server default one + return LocalDateTime + .parse( + resultSet.getString(columnIndex), + DATE_TIME_FORMAT + ); + } else if (columnTypeName.startsWith("DateTime")) { + Matcher matcher = PATTERN.matcher(columnTypeName); + if (!matcher.find() || matcher.groupCount() < 3) { + throw new IllegalArgumentException("Invalid Column Type '" + columnTypeName + "'"); + } + + return LocalDateTime + .parse( + resultSet.getString(columnIndex), + DATE_TIME_FORMAT + ) + .atZone(ZoneId.of(matcher.group(3))) + .withZoneSameInstant(zoneId); + } + + if (columnTypeName.equals("Int8")) { + Byte columnByte = (Byte) columnValue; + return columnByte.intValue(); + } + + if (columnTypeName.equals("Date")) { + return columnValue; + } + + if (columnTypeName.startsWith("Array(")) { + return columnValue; + } + + if (columnTypeName.startsWith("Tuple(")) { + return columnValue; + } + + if (columnTypeName.equals("IPv4")) { + Inet4Address col = (Inet4Address) columnValue; + return col.toString().substring(1); + } + + if (columnTypeName.equals("IPv6")) { + Inet6Address col = (Inet6Address) columnValue; + return col.toString().substring(1); + } + + return super.convert(columnIndex, resultSet); + } +} diff --git a/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Query.java b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Query.java new file mode 100644 index 00000000..bede9487 --- /dev/null +++ b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Query.java @@ -0,0 +1,60 @@ +package io.kestra.plugin.jdbc.dremio; + +import io.kestra.core.models.annotations.Example; +import io.kestra.core.models.annotations.Plugin; +import io.kestra.core.models.tasks.RunnableTask; +import io.kestra.core.runners.RunContext; +import io.kestra.plugin.jdbc.AbstractCellConverter; +import io.kestra.plugin.jdbc.AbstractJdbcQuery; +import io.kestra.plugin.jdbc.AutoCommitInterface; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.ToString; +import lombok.experimental.SuperBuilder; + +import java.sql.DriverManager; +import java.sql.SQLException; +import java.time.ZoneId; + +@SuperBuilder +@ToString +@EqualsAndHashCode +@Getter +@NoArgsConstructor +@Schema( + title = "Query a Dremio database." +) +@Plugin( + examples = { + @Example( + title = "Send a sql query to a Dremio database and fetch a row as outputs", + code = { + "url: jdbc:dremio:direct=sql.dremio.cloud:443;ssl=true;PROJECT_ID=sampleProjectId;", + "username: $token", + "password: samplePersonalAccessToken", + "sql: select user", + "fetchOne: true", + } + ) + } +) +public class Query extends AbstractJdbcQuery implements RunnableTask, AutoCommitInterface { + protected final Boolean autoCommit = true; + + @Override + protected AbstractCellConverter getCellConverter(ZoneId zoneId) { + return new DremioCellConverter(zoneId); + } + + @Override + public void registerDriver() throws SQLException { + DriverManager.registerDriver(new com.dremio.jdbc.Driver()); + } + + @Override + public Output run(RunContext runContext) throws Exception { + return super.run(runContext); + } +} diff --git a/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Trigger.java b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Trigger.java new file mode 100644 index 00000000..c3254acd --- /dev/null +++ b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Trigger.java @@ -0,0 +1,76 @@ +package io.kestra.plugin.jdbc.dremio; + +import io.kestra.core.models.annotations.Example; +import io.kestra.core.models.annotations.Plugin; +import io.kestra.core.runners.RunContext; +import io.kestra.plugin.jdbc.AbstractJdbcQuery; +import io.kestra.plugin.jdbc.AbstractJdbcTrigger; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.ToString; +import lombok.experimental.SuperBuilder; + +import java.sql.DriverManager; +import java.sql.SQLException; + +@SuperBuilder +@ToString +@EqualsAndHashCode +@Getter +@NoArgsConstructor +@Schema( + title = "Wait for query on a Dremio database." +) +@Plugin( + examples = { + @Example( + title = "Wait for a sql query to return results and iterate through rows", + full = true, + code = { + "id: jdbc-trigger", + "namespace: io.kestra.tests", + "", + "tasks:", + " - id: each", + " type: io.kestra.core.tasks.flows.EachSequential", + " tasks:", + " - id: return", + " type: io.kestra.core.tasks.debugs.Return", + " format: \"{{json(taskrun.value)}}\"", + " value: \"{{ trigger.rows }}\"", + "", + "triggers:", + " - id: watch", + " type: io.kestra.plugin.jdbc.clickhouse.Trigger", + " interval: \"PT5M\"", + " sql: \"SELECT * FROM my_table\"" + } + ) + } +) +public class Trigger extends AbstractJdbcTrigger { + @Override + protected AbstractJdbcQuery.Output runQuery(RunContext runContext) throws Exception { + var query = Query.builder() + .id(this.id) + .type(Query.class.getName()) + .url(this.getUrl()) + .username(this.getUsername()) + .password(this.getPassword()) + .timeZoneId(this.getTimeZoneId()) + .sql(this.getSql()) + .fetch(this.isFetch()) + .store(this.isStore()) + .fetchOne(this.isFetchOne()) + .additionalVars(this.additionalVars) + .build(); + return query.run(runContext); + } + + @Override + public void registerDriver() throws SQLException { + DriverManager.registerDriver(new com.dremio.jdbc.Driver()); + } +} diff --git a/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/package-info.java b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/package-info.java new file mode 100644 index 00000000..fa043e25 --- /dev/null +++ b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/package-info.java @@ -0,0 +1,7 @@ +@PluginSubGroup( + description = "This sub-group of plugins contains tasks for accessing the Dremio database.", + categories = PluginSubGroup.PluginCategory.DATABASE +) +package io.kestra.plugin.jdbc.dremio; + +import io.kestra.core.models.annotations.PluginSubGroup; \ No newline at end of file diff --git a/plugin-jdbc-dremio/src/main/resources/icons/io.kestra.plugin.jdbc.dremio.svg b/plugin-jdbc-dremio/src/main/resources/icons/io.kestra.plugin.jdbc.dremio.svg new file mode 100644 index 00000000..9d6ad9ea --- /dev/null +++ b/plugin-jdbc-dremio/src/main/resources/icons/io.kestra.plugin.jdbc.dremio.svg @@ -0,0 +1,54 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/plugin-jdbc-dremio/src/main/resources/icons/plugin-icon.svg b/plugin-jdbc-dremio/src/main/resources/icons/plugin-icon.svg new file mode 100644 index 00000000..9d6ad9ea --- /dev/null +++ b/plugin-jdbc-dremio/src/main/resources/icons/plugin-icon.svg @@ -0,0 +1,54 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/settings.gradle b/settings.gradle index 49eafbf9..336a3484 100644 --- a/settings.gradle +++ b/settings.gradle @@ -14,3 +14,4 @@ include 'plugin-jdbc-sqlserver' include 'plugin-jdbc-trino' include 'plugin-jdbc-vectorwise' include 'plugin-jdbc-vertica' +include 'plugin-jdbc-dremio' From 447c92889d52fade9485d6ceb1eb7fe4f945b704 Mon Sep 17 00:00:00 2001 From: iNikitaGricenko Date: Fri, 13 Oct 2023 17:01:19 +0300 Subject: [PATCH 2/9] change description and remove redundant converters --- .../jdbc/dremio/DremioCellConverter.java | 64 ------------------- .../io/kestra/plugin/jdbc/dremio/Query.java | 2 +- .../io/kestra/plugin/jdbc/dremio/Trigger.java | 2 +- 3 files changed, 2 insertions(+), 66 deletions(-) diff --git a/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/DremioCellConverter.java b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/DremioCellConverter.java index 0309b626..0fa081ec 100644 --- a/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/DremioCellConverter.java +++ b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/DremioCellConverter.java @@ -2,20 +2,14 @@ import io.kestra.plugin.jdbc.AbstractCellConverter; -import java.net.Inet4Address; -import java.net.Inet6Address; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; -import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; -import java.util.regex.Matcher; import java.util.regex.Pattern; public class DremioCellConverter extends AbstractCellConverter { - private static final Pattern PATTERN = Pattern.compile("DateTime(64)?\\((.*)'(.*)'\\)"); - private static final DateTimeFormatter DATE_TIME_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss[.SSSSSS][.SSSSS][.SSSS][.SSS][.SS][.S]"); public DremioCellConverter(ZoneId zoneId) { super(zoneId); @@ -23,64 +17,6 @@ public DremioCellConverter(ZoneId zoneId) { @Override public Object convertCell(int columnIndex, ResultSet resultSet, Connection connection) throws SQLException { - Object data = resultSet.getObject(columnIndex); - - if (data == null) { - return null; - } - - Object columnValue = resultSet.getObject(columnIndex); - String columnTypeName = resultSet.getMetaData().getColumnTypeName(columnIndex); - - if (columnTypeName.equals("DateTime")) { - // date with no TZ, we use the server default one - return LocalDateTime - .parse( - resultSet.getString(columnIndex), - DATE_TIME_FORMAT - ); - } else if (columnTypeName.startsWith("DateTime")) { - Matcher matcher = PATTERN.matcher(columnTypeName); - if (!matcher.find() || matcher.groupCount() < 3) { - throw new IllegalArgumentException("Invalid Column Type '" + columnTypeName + "'"); - } - - return LocalDateTime - .parse( - resultSet.getString(columnIndex), - DATE_TIME_FORMAT - ) - .atZone(ZoneId.of(matcher.group(3))) - .withZoneSameInstant(zoneId); - } - - if (columnTypeName.equals("Int8")) { - Byte columnByte = (Byte) columnValue; - return columnByte.intValue(); - } - - if (columnTypeName.equals("Date")) { - return columnValue; - } - - if (columnTypeName.startsWith("Array(")) { - return columnValue; - } - - if (columnTypeName.startsWith("Tuple(")) { - return columnValue; - } - - if (columnTypeName.equals("IPv4")) { - Inet4Address col = (Inet4Address) columnValue; - return col.toString().substring(1); - } - - if (columnTypeName.equals("IPv6")) { - Inet6Address col = (Inet6Address) columnValue; - return col.toString().substring(1); - } - return super.convert(columnIndex, resultSet); } } diff --git a/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Query.java b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Query.java index bede9487..334b67d9 100644 --- a/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Query.java +++ b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Query.java @@ -34,7 +34,7 @@ "url: jdbc:dremio:direct=sql.dremio.cloud:443;ssl=true;PROJECT_ID=sampleProjectId;", "username: $token", "password: samplePersonalAccessToken", - "sql: select user", + "sql: select * FROM source.database.table", "fetchOne: true", } ) diff --git a/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Trigger.java b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Trigger.java index c3254acd..d6012ef3 100644 --- a/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Trigger.java +++ b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Trigger.java @@ -45,7 +45,7 @@ " - id: watch", " type: io.kestra.plugin.jdbc.clickhouse.Trigger", " interval: \"PT5M\"", - " sql: \"SELECT * FROM my_table\"" + " sql: \"SELECT * FROM source.database.my_table\"" } ) } From 81f90067746a1dcca06395d3e73320374b48ee66 Mon Sep 17 00:00:00 2001 From: iNikitaGricenko Date: Mon, 16 Oct 2023 13:24:21 +0300 Subject: [PATCH 3/9] add arrow flight sql query task --- .../plugin/jdbc/dremio/ArrowFlightQuery.java | 62 +++++++++++++++++++ .../jdbc/dremio/DremioCellConverter.java | 2 - 2 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/ArrowFlightQuery.java diff --git a/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/ArrowFlightQuery.java b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/ArrowFlightQuery.java new file mode 100644 index 00000000..ee41cc94 --- /dev/null +++ b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/ArrowFlightQuery.java @@ -0,0 +1,62 @@ +package io.kestra.plugin.jdbc.dremio; + +import cfjd.org.apache.arrow.flight.sql.impl.FlightSql; +import io.kestra.core.models.annotations.Example; +import io.kestra.core.models.annotations.Plugin; +import io.kestra.core.models.tasks.RunnableTask; +import io.kestra.core.runners.RunContext; +import io.kestra.plugin.jdbc.AbstractCellConverter; +import io.kestra.plugin.jdbc.AbstractJdbcQuery; +import io.kestra.plugin.jdbc.AutoCommitInterface; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.ToString; +import lombok.experimental.SuperBuilder; +import org.apache.arrow.driver.jdbc.ArrowFlightJdbcDriver; + +import java.sql.DriverManager; +import java.sql.SQLException; +import java.time.ZoneId; + +@SuperBuilder +@ToString +@EqualsAndHashCode +@Getter +@NoArgsConstructor +@Schema( + title = "Query a Dremio database with Arrow Flight SQL." +) +@Plugin( + examples = { + @Example( + title = "Send a sql query to a Dremio database and fetch a row as outputs", + code = { + "url: jdbc:arrow-flight-sql://data.dremio.cloud:443", + "username: $token", + "password: samplePersonalAccessToken", + "sql: select * FROM source.database.table", + "fetchOne: true", + } + ) + } +) +public class ArrowFlightQuery extends AbstractJdbcQuery implements RunnableTask, AutoCommitInterface { + protected final Boolean autoCommit = true; + + @Override + protected AbstractCellConverter getCellConverter(ZoneId zoneId) { + return new DremioCellConverter(zoneId); + } + + @Override + public void registerDriver() throws SQLException { + DriverManager.registerDriver(new ArrowFlightJdbcDriver()); + } + + @Override + public Output run(RunContext runContext) throws Exception { + return super.run(runContext); + } +} diff --git a/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/DremioCellConverter.java b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/DremioCellConverter.java index 0fa081ec..23f78c58 100644 --- a/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/DremioCellConverter.java +++ b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/DremioCellConverter.java @@ -6,8 +6,6 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.regex.Pattern; public class DremioCellConverter extends AbstractCellConverter { From 35ff8489a7fa071bbf5c004b64879cb90e4cc6f8 Mon Sep 17 00:00:00 2001 From: iNikitaGricenko Date: Mon, 16 Oct 2023 15:48:36 +0300 Subject: [PATCH 4/9] dremio store variable fix --- .../plugin/jdbc/dremio/ArrowFlightQuery.java | 62 ------------------- .../io/kestra/plugin/jdbc/dremio/Query.java | 34 ++++++++-- .../io/kestra/plugin/jdbc/dremio/Trigger.java | 2 +- 3 files changed, 31 insertions(+), 67 deletions(-) delete mode 100644 plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/ArrowFlightQuery.java diff --git a/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/ArrowFlightQuery.java b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/ArrowFlightQuery.java deleted file mode 100644 index ee41cc94..00000000 --- a/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/ArrowFlightQuery.java +++ /dev/null @@ -1,62 +0,0 @@ -package io.kestra.plugin.jdbc.dremio; - -import cfjd.org.apache.arrow.flight.sql.impl.FlightSql; -import io.kestra.core.models.annotations.Example; -import io.kestra.core.models.annotations.Plugin; -import io.kestra.core.models.tasks.RunnableTask; -import io.kestra.core.runners.RunContext; -import io.kestra.plugin.jdbc.AbstractCellConverter; -import io.kestra.plugin.jdbc.AbstractJdbcQuery; -import io.kestra.plugin.jdbc.AutoCommitInterface; -import io.swagger.v3.oas.annotations.media.Schema; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.ToString; -import lombok.experimental.SuperBuilder; -import org.apache.arrow.driver.jdbc.ArrowFlightJdbcDriver; - -import java.sql.DriverManager; -import java.sql.SQLException; -import java.time.ZoneId; - -@SuperBuilder -@ToString -@EqualsAndHashCode -@Getter -@NoArgsConstructor -@Schema( - title = "Query a Dremio database with Arrow Flight SQL." -) -@Plugin( - examples = { - @Example( - title = "Send a sql query to a Dremio database and fetch a row as outputs", - code = { - "url: jdbc:arrow-flight-sql://data.dremio.cloud:443", - "username: $token", - "password: samplePersonalAccessToken", - "sql: select * FROM source.database.table", - "fetchOne: true", - } - ) - } -) -public class ArrowFlightQuery extends AbstractJdbcQuery implements RunnableTask, AutoCommitInterface { - protected final Boolean autoCommit = true; - - @Override - protected AbstractCellConverter getCellConverter(ZoneId zoneId) { - return new DremioCellConverter(zoneId); - } - - @Override - public void registerDriver() throws SQLException { - DriverManager.registerDriver(new ArrowFlightJdbcDriver()); - } - - @Override - public Output run(RunContext runContext) throws Exception { - return super.run(runContext); - } -} diff --git a/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Query.java b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Query.java index 334b67d9..72a4f05c 100644 --- a/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Query.java +++ b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Query.java @@ -4,19 +4,22 @@ import io.kestra.core.models.annotations.Plugin; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; +import io.kestra.core.serializers.FileSerde; import io.kestra.plugin.jdbc.AbstractCellConverter; import io.kestra.plugin.jdbc.AbstractJdbcQuery; import io.kestra.plugin.jdbc.AutoCommitInterface; import io.swagger.v3.oas.annotations.media.Schema; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.ToString; +import lombok.*; import lombok.experimental.SuperBuilder; +import java.io.*; import java.sql.DriverManager; import java.sql.SQLException; import java.time.ZoneId; +import java.util.List; +import java.util.Map; + +import static io.kestra.core.utils.Rethrow.throwConsumer; @SuperBuilder @ToString @@ -37,11 +40,15 @@ "sql: select * FROM source.database.table", "fetchOne: true", } + ), + @Example( + ) } ) public class Query extends AbstractJdbcQuery implements RunnableTask, AutoCommitInterface { protected final Boolean autoCommit = true; + private boolean store = false; @Override protected AbstractCellConverter getCellConverter(ZoneId zoneId) { @@ -55,6 +62,25 @@ public void registerDriver() throws SQLException { @Override public Output run(RunContext runContext) throws Exception { + if (store) { + Output output = super.run(runContext); + + File tempFile = runContext.tempFile(".ion").toFile(); + saveIntoFile(output.getRows(), tempFile); + + return Output.builder() + .size(output.getSize()) + .uri(runContext.putTempFile(tempFile)) + .build(); + } return super.run(runContext); } + + private void saveIntoFile(List> rows, File tempFile) throws IOException { + try (OutputStream output = new FileOutputStream(tempFile)) { + rows.forEach(throwConsumer(row -> FileSerde.write(output, row))); + } + } + + } diff --git a/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Trigger.java b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Trigger.java index d6012ef3..2b0b190c 100644 --- a/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Trigger.java +++ b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Trigger.java @@ -61,8 +61,8 @@ protected AbstractJdbcQuery.Output runQuery(RunContext runContext) throws Except .password(this.getPassword()) .timeZoneId(this.getTimeZoneId()) .sql(this.getSql()) - .fetch(this.isFetch()) .store(this.isStore()) + .fetch(this.isStore() || this.isFetch()) .fetchOne(this.isFetchOne()) .additionalVars(this.additionalVars) .build(); From 5b40edbdbbb218221278fb963e86cf3e4958e826 Mon Sep 17 00:00:00 2001 From: iNikitaGricenko Date: Mon, 16 Oct 2023 17:14:11 +0300 Subject: [PATCH 5/9] dremio autocommit --- .../io/kestra/plugin/jdbc/dremio/Query.java | 39 +++---------------- .../io/kestra/plugin/jdbc/dremio/Trigger.java | 2 +- 2 files changed, 6 insertions(+), 35 deletions(-) diff --git a/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Query.java b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Query.java index 72a4f05c..5f63060f 100644 --- a/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Query.java +++ b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Query.java @@ -4,22 +4,18 @@ import io.kestra.core.models.annotations.Plugin; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; -import io.kestra.core.serializers.FileSerde; import io.kestra.plugin.jdbc.AbstractCellConverter; import io.kestra.plugin.jdbc.AbstractJdbcQuery; -import io.kestra.plugin.jdbc.AutoCommitInterface; import io.swagger.v3.oas.annotations.media.Schema; -import lombok.*; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.ToString; import lombok.experimental.SuperBuilder; -import java.io.*; import java.sql.DriverManager; import java.sql.SQLException; import java.time.ZoneId; -import java.util.List; -import java.util.Map; - -import static io.kestra.core.utils.Rethrow.throwConsumer; @SuperBuilder @ToString @@ -40,16 +36,10 @@ "sql: select * FROM source.database.table", "fetchOne: true", } - ), - @Example( - ) } ) -public class Query extends AbstractJdbcQuery implements RunnableTask, AutoCommitInterface { - protected final Boolean autoCommit = true; - private boolean store = false; - +public class Query extends AbstractJdbcQuery implements RunnableTask { @Override protected AbstractCellConverter getCellConverter(ZoneId zoneId) { return new DremioCellConverter(zoneId); @@ -62,25 +52,6 @@ public void registerDriver() throws SQLException { @Override public Output run(RunContext runContext) throws Exception { - if (store) { - Output output = super.run(runContext); - - File tempFile = runContext.tempFile(".ion").toFile(); - saveIntoFile(output.getRows(), tempFile); - - return Output.builder() - .size(output.getSize()) - .uri(runContext.putTempFile(tempFile)) - .build(); - } return super.run(runContext); } - - private void saveIntoFile(List> rows, File tempFile) throws IOException { - try (OutputStream output = new FileOutputStream(tempFile)) { - rows.forEach(throwConsumer(row -> FileSerde.write(output, row))); - } - } - - } diff --git a/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Trigger.java b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Trigger.java index 2b0b190c..882b3dfc 100644 --- a/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Trigger.java +++ b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Trigger.java @@ -62,7 +62,7 @@ protected AbstractJdbcQuery.Output runQuery(RunContext runContext) throws Except .timeZoneId(this.getTimeZoneId()) .sql(this.getSql()) .store(this.isStore()) - .fetch(this.isStore() || this.isFetch()) + .fetch(this.isFetch()) .fetchOne(this.isFetchOne()) .additionalVars(this.additionalVars) .build(); From 0d9ad217384a33a0b353f8f4e4efe5ee38e65847 Mon Sep 17 00:00:00 2001 From: iNikitaGricenko Date: Wed, 18 Oct 2023 15:55:56 +0300 Subject: [PATCH 6/9] add arrow flight sql query --- .../plugin/jdbc/dremio/ArrowFlightQuery.java | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/ArrowFlightQuery.java diff --git a/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/ArrowFlightQuery.java b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/ArrowFlightQuery.java new file mode 100644 index 00000000..90cb3a0c --- /dev/null +++ b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/ArrowFlightQuery.java @@ -0,0 +1,68 @@ +package io.kestra.plugin.jdbc.dremio; + +import io.kestra.core.models.annotations.Example; +import io.kestra.core.models.annotations.Plugin; +import io.kestra.core.models.tasks.RunnableTask; +import io.kestra.core.runners.RunContext; +import io.kestra.plugin.jdbc.AbstractCellConverter; +import io.kestra.plugin.jdbc.AbstractJdbcQuery; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.ToString; +import lombok.experimental.SuperBuilder; +import org.apache.arrow.driver.jdbc.ArrowFlightJdbcDriver; + +import java.sql.DriverManager; +import java.sql.SQLException; +import java.time.ZoneId; + +@SuperBuilder +@ToString +@EqualsAndHashCode +@Getter +@NoArgsConstructor +@Schema( + title = "Query a Dremio database through Apache Arrow Flight sql driver." +) +@Plugin( + examples = { + @Example( + title = "Send a sql query to a Dremio direct database and fetch a row as outputs using Apache Arrow Flight sql driver", + code = { + "url: jdbc:arrow-flight-sql://localhost:31010/?useEncryption=false", + "username: dremio", + "password: dremio123", + "sql: select * FROM \"postgres.public\".departments", + "fetch: true", + } + ), + @Example( + title = "Send a sql query to a Dremio coordinator and fetch a row as outputs using Apache Arrow Flight sql driver", + code = { + "url: jdbc:arrow-flight-sql://dremio-coordinator:32010/?schema=postgres.public", + "username: $token", + "password: samplePersonalAccessToken", + "sql: select * FROM departments", + "fetch: true", + } + ) + } +) +public class ArrowFlightQuery extends AbstractJdbcQuery implements RunnableTask { + @Override + protected AbstractCellConverter getCellConverter(ZoneId zoneId) { + return new DremioCellConverter(zoneId); + } + + @Override + public void registerDriver() throws SQLException { + DriverManager.registerDriver(new ArrowFlightJdbcDriver()); + } + + @Override + public Output run(RunContext runContext) throws Exception { + return super.run(runContext); + } +} From 27e90309afaf5ddf6ae7790b66584e5d310ab9a9 Mon Sep 17 00:00:00 2001 From: iNikitaGricenko Date: Mon, 23 Oct 2023 15:31:27 +0300 Subject: [PATCH 7/9] split apache arrow flight and dremio plugins --- plugin-jdbc-arrow-flight/build.gradle | 20 +++++ .../arrowflight/ArrowFlightCellConverter.java | 20 +++++ .../kestra/plugin/jdbc/arrowflight/Query.java | 12 +-- .../plugin/jdbc/arrowflight/Trigger.java | 78 +++++++++++++++++++ .../plugin/jdbc/arrowflight/package-info.java | 7 ++ .../io.kestra.plugin.jdbc.arrow-flight.svg | 26 +++++++ .../src/main/resources/icons/plugin-icon.svg | 26 +++++++ plugin-jdbc-dremio/build.gradle | 1 - .../io/kestra/plugin/jdbc/dremio/Trigger.java | 2 +- settings.gradle | 2 + 10 files changed, 186 insertions(+), 8 deletions(-) create mode 100644 plugin-jdbc-arrow-flight/build.gradle create mode 100644 plugin-jdbc-arrow-flight/src/main/java/io/kestra/plugin/jdbc/arrowflight/ArrowFlightCellConverter.java rename plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/ArrowFlightQuery.java => plugin-jdbc-arrow-flight/src/main/java/io/kestra/plugin/jdbc/arrowflight/Query.java (82%) create mode 100644 plugin-jdbc-arrow-flight/src/main/java/io/kestra/plugin/jdbc/arrowflight/Trigger.java create mode 100644 plugin-jdbc-arrow-flight/src/main/java/io/kestra/plugin/jdbc/arrowflight/package-info.java create mode 100644 plugin-jdbc-arrow-flight/src/main/resources/icons/io.kestra.plugin.jdbc.arrow-flight.svg create mode 100644 plugin-jdbc-arrow-flight/src/main/resources/icons/plugin-icon.svg diff --git a/plugin-jdbc-arrow-flight/build.gradle b/plugin-jdbc-arrow-flight/build.gradle new file mode 100644 index 00000000..d5ced12d --- /dev/null +++ b/plugin-jdbc-arrow-flight/build.gradle @@ -0,0 +1,20 @@ +project.description = 'Query Dremio databases using the Kestra Apache Arrow Flight JDBC plugin.' + +jar { + manifest { + attributes( + "X-Kestra-Name": project.name, + "X-Kestra-Title": "Arrow Flight", + "X-Kestra-Group": project.group + ".jdbc.arrowflight", + "X-Kestra-Description": project.description, + "X-Kestra-Version": project.version + ) + } +} + +dependencies { + implementation("org.apache.arrow:flight-sql-jdbc-driver:12.0.1") + implementation project(':plugin-jdbc') + + testImplementation project(':plugin-jdbc').sourceSets.test.output +} \ No newline at end of file diff --git a/plugin-jdbc-arrow-flight/src/main/java/io/kestra/plugin/jdbc/arrowflight/ArrowFlightCellConverter.java b/plugin-jdbc-arrow-flight/src/main/java/io/kestra/plugin/jdbc/arrowflight/ArrowFlightCellConverter.java new file mode 100644 index 00000000..6861c6c0 --- /dev/null +++ b/plugin-jdbc-arrow-flight/src/main/java/io/kestra/plugin/jdbc/arrowflight/ArrowFlightCellConverter.java @@ -0,0 +1,20 @@ +package io.kestra.plugin.jdbc.arrowflight; + +import io.kestra.plugin.jdbc.AbstractCellConverter; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.ZoneId; + +public class ArrowFlightCellConverter extends AbstractCellConverter { + + public ArrowFlightCellConverter(ZoneId zoneId) { + super(zoneId); + } + + @Override + public Object convertCell(int columnIndex, ResultSet resultSet, Connection connection) throws SQLException { + return super.convert(columnIndex, resultSet); + } +} diff --git a/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/ArrowFlightQuery.java b/plugin-jdbc-arrow-flight/src/main/java/io/kestra/plugin/jdbc/arrowflight/Query.java similarity index 82% rename from plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/ArrowFlightQuery.java rename to plugin-jdbc-arrow-flight/src/main/java/io/kestra/plugin/jdbc/arrowflight/Query.java index 90cb3a0c..c4d2c512 100644 --- a/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/ArrowFlightQuery.java +++ b/plugin-jdbc-arrow-flight/src/main/java/io/kestra/plugin/jdbc/arrowflight/Query.java @@ -1,4 +1,4 @@ -package io.kestra.plugin.jdbc.dremio; +package io.kestra.plugin.jdbc.arrowflight; import io.kestra.core.models.annotations.Example; import io.kestra.core.models.annotations.Plugin; @@ -24,12 +24,12 @@ @Getter @NoArgsConstructor @Schema( - title = "Query a Dremio database through Apache Arrow Flight sql driver." + title = "Query a databases through Apache Arrow Flight SQL driver." ) @Plugin( examples = { @Example( - title = "Send a sql query to a Dremio direct database and fetch a row as outputs using Apache Arrow Flight sql driver", + title = "Send a sql query to a Dremio direct database and fetch a row as outputs using Apache Arrow Flight SQL driver", code = { "url: jdbc:arrow-flight-sql://localhost:31010/?useEncryption=false", "username: dremio", @@ -39,7 +39,7 @@ } ), @Example( - title = "Send a sql query to a Dremio coordinator and fetch a row as outputs using Apache Arrow Flight sql driver", + title = "Send a sql query to a Dremio coordinator and fetch a row as outputs using Apache Arrow Flight SQL driver", code = { "url: jdbc:arrow-flight-sql://dremio-coordinator:32010/?schema=postgres.public", "username: $token", @@ -50,10 +50,10 @@ ) } ) -public class ArrowFlightQuery extends AbstractJdbcQuery implements RunnableTask { +public class Query extends AbstractJdbcQuery implements RunnableTask { @Override protected AbstractCellConverter getCellConverter(ZoneId zoneId) { - return new DremioCellConverter(zoneId); + return new ArrowFlightCellConverter(zoneId); } @Override diff --git a/plugin-jdbc-arrow-flight/src/main/java/io/kestra/plugin/jdbc/arrowflight/Trigger.java b/plugin-jdbc-arrow-flight/src/main/java/io/kestra/plugin/jdbc/arrowflight/Trigger.java new file mode 100644 index 00000000..a90a32fc --- /dev/null +++ b/plugin-jdbc-arrow-flight/src/main/java/io/kestra/plugin/jdbc/arrowflight/Trigger.java @@ -0,0 +1,78 @@ +package io.kestra.plugin.jdbc.arrowflight; + +import io.kestra.core.models.annotations.Example; +import io.kestra.core.models.annotations.Plugin; +import io.kestra.core.runners.RunContext; +import io.kestra.plugin.jdbc.AbstractJdbcQuery; +import io.kestra.plugin.jdbc.AbstractJdbcTrigger; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.ToString; +import lombok.experimental.SuperBuilder; +import org.apache.arrow.driver.jdbc.ArrowFlightJdbcDriver; + +import java.sql.DriverManager; +import java.sql.SQLException; + +@SuperBuilder +@ToString +@EqualsAndHashCode +@Getter +@NoArgsConstructor +@Schema( + title = "Wait for query on a Arrow Flight database." +) +@Plugin( + examples = { + @Example( + title = "Wait for a sql query to return results and iterate through rows", + full = true, + code = { + "id: jdbc-trigger", + "namespace: io.kestra.tests", + "", + "tasks:", + " - id: each", + " type: io.kestra.core.tasks.flows.EachSequential", + " tasks:", + " - id: return", + " type: io.kestra.core.tasks.debugs.Return", + " format: \"{{json(taskrun.value)}}\"", + " value: \"{{ trigger.rows }}\"", + "", + "triggers:", + " - id: watch", + " type: io.kestra.plugin.jdbc.arrowflight.Trigger", + " url: jdbc:arrow-flight-sql://dremio-coordinator:32010/?schema=postgres.public", + " interval: \"PT5M\"", + " sql: \"SELECT * FROM my_table\"" + } + ) + } +) +public class Trigger extends AbstractJdbcTrigger { + @Override + protected AbstractJdbcQuery.Output runQuery(RunContext runContext) throws Exception { + var query = Query.builder() + .id(this.id) + .type(Query.class.getName()) + .url(this.getUrl()) + .username(this.getUsername()) + .password(this.getPassword()) + .timeZoneId(this.getTimeZoneId()) + .sql(this.getSql()) + .store(this.isStore()) + .fetch(this.isFetch()) + .fetchOne(this.isFetchOne()) + .additionalVars(this.additionalVars) + .build(); + return query.run(runContext); + } + + @Override + public void registerDriver() throws SQLException { + DriverManager.registerDriver(new ArrowFlightJdbcDriver()); + } +} diff --git a/plugin-jdbc-arrow-flight/src/main/java/io/kestra/plugin/jdbc/arrowflight/package-info.java b/plugin-jdbc-arrow-flight/src/main/java/io/kestra/plugin/jdbc/arrowflight/package-info.java new file mode 100644 index 00000000..bfa8587e --- /dev/null +++ b/plugin-jdbc-arrow-flight/src/main/java/io/kestra/plugin/jdbc/arrowflight/package-info.java @@ -0,0 +1,7 @@ +@PluginSubGroup( + description = "This sub-group of plugins contains tasks for accessing the databases through Apache Arrow Flight.", + categories = PluginSubGroup.PluginCategory.DATABASE +) +package io.kestra.plugin.jdbc.arrowflight; + +import io.kestra.core.models.annotations.PluginSubGroup; \ No newline at end of file diff --git a/plugin-jdbc-arrow-flight/src/main/resources/icons/io.kestra.plugin.jdbc.arrow-flight.svg b/plugin-jdbc-arrow-flight/src/main/resources/icons/io.kestra.plugin.jdbc.arrow-flight.svg new file mode 100644 index 00000000..1cd036e8 --- /dev/null +++ b/plugin-jdbc-arrow-flight/src/main/resources/icons/io.kestra.plugin.jdbc.arrow-flight.svg @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/plugin-jdbc-arrow-flight/src/main/resources/icons/plugin-icon.svg b/plugin-jdbc-arrow-flight/src/main/resources/icons/plugin-icon.svg new file mode 100644 index 00000000..1cd036e8 --- /dev/null +++ b/plugin-jdbc-arrow-flight/src/main/resources/icons/plugin-icon.svg @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/plugin-jdbc-dremio/build.gradle b/plugin-jdbc-dremio/build.gradle index 6d9a8809..7642ed8f 100644 --- a/plugin-jdbc-dremio/build.gradle +++ b/plugin-jdbc-dremio/build.gradle @@ -14,7 +14,6 @@ jar { dependencies { implementation("com.dremio.distribution:dremio-jdbc-driver:3.0.6-201812082352540436-1f684f9") - implementation("org.apache.arrow:flight-sql-jdbc-driver:12.0.1") implementation project(':plugin-jdbc') testImplementation project(':plugin-jdbc').sourceSets.test.output diff --git a/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Trigger.java b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Trigger.java index 882b3dfc..3f7edcf2 100644 --- a/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Trigger.java +++ b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Trigger.java @@ -43,7 +43,7 @@ "", "triggers:", " - id: watch", - " type: io.kestra.plugin.jdbc.clickhouse.Trigger", + " type: io.kestra.plugin.jdbc.dremio.Trigger", " interval: \"PT5M\"", " sql: \"SELECT * FROM source.database.my_table\"" } diff --git a/settings.gradle b/settings.gradle index 336a3484..c1ea65f0 100644 --- a/settings.gradle +++ b/settings.gradle @@ -15,3 +15,5 @@ include 'plugin-jdbc-trino' include 'plugin-jdbc-vectorwise' include 'plugin-jdbc-vertica' include 'plugin-jdbc-dremio' +include 'plugin-jdbc-arrow-flight' + From 8cd3514f26a0f28dfd8eb4f9111bbfed8b1afd7a Mon Sep 17 00:00:00 2001 From: Anna Geller Date: Thu, 26 Oct 2023 15:15:03 +0200 Subject: [PATCH 8/9] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Loïc Mathieu --- plugin-jdbc-arrow-flight/build.gradle | 4 ++-- .../main/java/io/kestra/plugin/jdbc/arrowflight/Query.java | 6 +++--- .../java/io/kestra/plugin/jdbc/arrowflight/Trigger.java | 2 +- plugin-jdbc-dremio/build.gradle | 2 +- .../src/main/java/io/kestra/plugin/jdbc/dremio/Query.java | 2 +- .../src/main/java/io/kestra/plugin/jdbc/dremio/Trigger.java | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/plugin-jdbc-arrow-flight/build.gradle b/plugin-jdbc-arrow-flight/build.gradle index d5ced12d..89a0ea5f 100644 --- a/plugin-jdbc-arrow-flight/build.gradle +++ b/plugin-jdbc-arrow-flight/build.gradle @@ -1,10 +1,10 @@ -project.description = 'Query Dremio databases using the Kestra Apache Arrow Flight JDBC plugin.' +project.description = 'Query a compatible database using the Kestra Apache Arrow Flight SQL JDBC plugin.' jar { manifest { attributes( "X-Kestra-Name": project.name, - "X-Kestra-Title": "Arrow Flight", + "X-Kestra-Title": "Arrow Flight SQL", "X-Kestra-Group": project.group + ".jdbc.arrowflight", "X-Kestra-Description": project.description, "X-Kestra-Version": project.version diff --git a/plugin-jdbc-arrow-flight/src/main/java/io/kestra/plugin/jdbc/arrowflight/Query.java b/plugin-jdbc-arrow-flight/src/main/java/io/kestra/plugin/jdbc/arrowflight/Query.java index c4d2c512..a313f2b9 100644 --- a/plugin-jdbc-arrow-flight/src/main/java/io/kestra/plugin/jdbc/arrowflight/Query.java +++ b/plugin-jdbc-arrow-flight/src/main/java/io/kestra/plugin/jdbc/arrowflight/Query.java @@ -24,17 +24,17 @@ @Getter @NoArgsConstructor @Schema( - title = "Query a databases through Apache Arrow Flight SQL driver." + title = "Query a database through Apache Arrow Flight SQL driver." ) @Plugin( examples = { @Example( - title = "Send a sql query to a Dremio direct database and fetch a row as outputs using Apache Arrow Flight SQL driver", + title = "Send a SQL query to a database and fetch a row as outputs using Apache Arrow Flight SQL driver", code = { "url: jdbc:arrow-flight-sql://localhost:31010/?useEncryption=false", "username: dremio", "password: dremio123", - "sql: select * FROM \"postgres.public\".departments", + "sql: select * FROM departments", "fetch: true", } ), diff --git a/plugin-jdbc-arrow-flight/src/main/java/io/kestra/plugin/jdbc/arrowflight/Trigger.java b/plugin-jdbc-arrow-flight/src/main/java/io/kestra/plugin/jdbc/arrowflight/Trigger.java index a90a32fc..fe54c148 100644 --- a/plugin-jdbc-arrow-flight/src/main/java/io/kestra/plugin/jdbc/arrowflight/Trigger.java +++ b/plugin-jdbc-arrow-flight/src/main/java/io/kestra/plugin/jdbc/arrowflight/Trigger.java @@ -27,7 +27,7 @@ @Plugin( examples = { @Example( - title = "Wait for a sql query to return results and iterate through rows", + title = "Wait for a SQL query to return results and iterate through rows", full = true, code = { "id: jdbc-trigger", diff --git a/plugin-jdbc-dremio/build.gradle b/plugin-jdbc-dremio/build.gradle index 7642ed8f..ab31a65f 100644 --- a/plugin-jdbc-dremio/build.gradle +++ b/plugin-jdbc-dremio/build.gradle @@ -1,4 +1,4 @@ -project.description = 'Query Dremio databases using the Kestra JDBC plugin.' +project.description = 'Query Dremio database using the Kestra JDBC plugin.' jar { manifest { diff --git a/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Query.java b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Query.java index 5f63060f..8ef44c76 100644 --- a/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Query.java +++ b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Query.java @@ -28,7 +28,7 @@ @Plugin( examples = { @Example( - title = "Send a sql query to a Dremio database and fetch a row as outputs", + title = "Send a SQL query to a Dremio database and fetch a row as outputs", code = { "url: jdbc:dremio:direct=sql.dremio.cloud:443;ssl=true;PROJECT_ID=sampleProjectId;", "username: $token", diff --git a/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Trigger.java b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Trigger.java index 3f7edcf2..659d6f35 100644 --- a/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Trigger.java +++ b/plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Trigger.java @@ -26,7 +26,7 @@ @Plugin( examples = { @Example( - title = "Wait for a sql query to return results and iterate through rows", + title = "Wait for a SQL query to return results and iterate through rows", full = true, code = { "id: jdbc-trigger", From f0766f10ffe564aa442eb7c2fd5988907eb1e9ee Mon Sep 17 00:00:00 2001 From: Anna Geller Date: Thu, 26 Oct 2023 15:15:25 +0200 Subject: [PATCH 9/9] Update plugin-jdbc-arrow-flight/src/main/java/io/kestra/plugin/jdbc/arrowflight/package-info.java MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Loïc Mathieu --- .../java/io/kestra/plugin/jdbc/arrowflight/package-info.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin-jdbc-arrow-flight/src/main/java/io/kestra/plugin/jdbc/arrowflight/package-info.java b/plugin-jdbc-arrow-flight/src/main/java/io/kestra/plugin/jdbc/arrowflight/package-info.java index bfa8587e..5f7b0b04 100644 --- a/plugin-jdbc-arrow-flight/src/main/java/io/kestra/plugin/jdbc/arrowflight/package-info.java +++ b/plugin-jdbc-arrow-flight/src/main/java/io/kestra/plugin/jdbc/arrowflight/package-info.java @@ -1,5 +1,5 @@ @PluginSubGroup( - description = "This sub-group of plugins contains tasks for accessing the databases through Apache Arrow Flight.", + description = "This sub-group of plugins contains tasks for accessing a databases through Apache Arrow Flight SQL.", categories = PluginSubGroup.PluginCategory.DATABASE ) package io.kestra.plugin.jdbc.arrowflight;