Skip to content

Commit

Permalink
feat: Correct plugin-jdbc docs for multiple data sources (#205)
Browse files Browse the repository at this point in the history 
* Correct plugin-jdbc docs for multiple data sources

* Update plugin-jdbc-duckdb/src/main/java/io/kestra/plugin/jdbc/duckdb/Query.java

---------

Co-authored-by: Shruti Mantri <smantri@moveworks.ai>
Co-authored-by: Loïc Mathieu <loikeseke@gmail.com>
  • Loading branch information
@shrutimantri @smantri-moveworks @loicmathieu
3 people authored Jan 2, 2024
1 parent bfb16a3 commit 85ea919
Show file tree
Hide file tree
Showing 16 changed files with 63 additions and 48 deletions.
4 changes: 2 additions & 2 deletions plugin-jdbc-arrow-flight/src/main/java/io/kestra/plugin/jdbc/arrowflight/Query.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
@Plugin(
examples = {
@Example(
title = "Send a SQL query to a database and fetch row(s) using Apache Arrow Flight SQL driver",
title = "Send a SQL query to a database and fetch row(s) using Apache Arrow Flight SQL driver.",
code = {
"url: jdbc:arrow-flight-sql://localhost:31010/?useEncryption=false",
"username: db_user",
Expand All @@ -39,7 +39,7 @@
}
),
@Example(
title = "Send a sql query to a Dremio coordinator and fetch rows as outputs using Apache Arrow Flight SQL driver",
title = "Send a sql query to a Dremio coordinator and fetch rows as outputs using Apache Arrow Flight SQL driver.",
code = {
"url: jdbc:arrow-flight-sql://dremio-coordinator:32010/?schema=postgres.public",
"username: dremio_user",
Expand Down
4 changes: 2 additions & 2 deletions plugin-jdbc-arrow-flight/src/main/java/io/kestra/plugin/jdbc/arrowflight/Trigger.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
@Plugin(
examples = {
@Example(
title = "Wait for a SQL query to return results and iterate through rows",
title = "Wait for a SQL query to return results, and then iterate through rows.",
full = true,
code = {
"id: jdbc-trigger",
Expand All @@ -39,7 +39,7 @@
" tasks:",
" - id: return",
" type: io.kestra.core.tasks.debugs.Return",
" format: \"{{json(taskrun.value)}}\"",
" format: \"{{ json(taskrun.value) }}\"",
" value: \"{{ trigger.rows }}\"",
"",
"triggers:",
Expand Down
8 changes: 4 additions & 4 deletions plugin-jdbc-clickhouse/src/main/java/io/kestra/plugin/jdbc/clickhouse/BulkInsert.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,21 +31,21 @@
@Plugin(
examples = {
@Example(
title = "Insert rows from another table to a Clickhouse database using asynchronous inserts",
title = "Insert rows from another table to a Clickhouse database using asynchronous inserts.",
code = {
"from: \"{{ outputs.query.uri }}\"",
"url: jdbc:clickhouse://127.0.0.1:56982/",
"username: clickhouse",
"username: ch_user",
"password: ch_passwd",
"sql: INSERT INTO YourTable SETTINGS async_insert=1, wait_for_async_insert=1 values( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )"
}
),
@Example(
title = "Insert data into specific columns via a SQL query to a ClickHouse database using asynchronous inserts",
title = "Insert data into specific columns via a SQL query to a ClickHouse database using asynchronous inserts.",
code = {
"from: \"{{ outputs.query.uri }}\"",
"url: jdbc:clickhouse://127.0.0.1:56982/",
"username: clickhouse",
"username: ch_user",
"password: ch_passwd",
"sql: INSERT INTO YourTable ( field1, field2, field3 ) SETTINGS async_insert=1, wait_for_async_insert=1 values( ?, ?, ? )"
}
Expand Down
2 changes: 1 addition & 1 deletion ...bc-clickhouse/src/main/java/io/kestra/plugin/jdbc/clickhouse/ClickHouseCellConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ public Object convertCell(int columnIndex, ResultSet rs, Connection connection)
} else if (columnTypeName.startsWith("DateTime")) {
Matcher matcher = PATTERN.matcher(columnTypeName);
if (!matcher.find() || matcher.groupCount() < 3) {
throw new IllegalArgumentException("Invalid Column Type '" + columnTypeName + "'");
throw new IllegalArgumentException("Invalid column type '" + columnTypeName + "'");
}

return LocalDateTime
Expand Down
4 changes: 2 additions & 2 deletions plugin-jdbc-clickhouse/src/main/java/io/kestra/plugin/jdbc/clickhouse/Query.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,10 @@
@Plugin(
examples = {
@Example(
title = "Send a sql query to a Clickhouse database and fetch a row as outputs",
title = "Send a sql query to a Clickhouse database and fetch a row as output.",
code = {
"url: jdbc:clickhouse://127.0.0.1:56982/",
"username: clickhouse",
"username: ch_user",
"password: ch_passwd",
"sql: select * from clickhouse_types",
"fetchOne: true",
Expand Down
7 changes: 5 additions & 2 deletions plugin-jdbc-clickhouse/src/main/java/io/kestra/plugin/jdbc/clickhouse/Trigger.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
@Plugin(
examples = {
@Example(
title = "Wait for a sql query to return results and iterate through rows",
title = "Wait for a sql query to return results, and then iterate through rows.",
full = true,
code = {
"id: jdbc-trigger",
Expand All @@ -38,13 +38,16 @@
" tasks:",
" - id: return",
" type: io.kestra.core.tasks.debugs.Return",
" format: \"{{json(taskrun.value)}}\"",
" format: \"{{ json(taskrun.value) }}\"",
" value: \"{{ trigger.rows }}\"",
"",
"triggers:",
" - id: watch",
" type: io.kestra.plugin.jdbc.clickhouse.Trigger",
" interval: \"PT5M\"",
" url: jdbc:clickhouse://127.0.0.1:56982/",
" username: ch_user",
" password: ch_passwd",
" sql: \"SELECT * FROM my_table\""
}
)
Expand Down
2 changes: 1 addition & 1 deletion plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Query.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
@Plugin(
examples = {
@Example(
title = "Send a SQL query to a Dremio database and fetch a row as outputs",
title = "Send a SQL query to a Dremio database and fetch a row as output.",
code = {
"url: jdbc:dremio:direct=sql.dremio.cloud:443;ssl=true;PROJECT_ID=sampleProjectId;",
"username: $token",
Expand Down
7 changes: 5 additions & 2 deletions plugin-jdbc-dremio/src/main/java/io/kestra/plugin/jdbc/dremio/Trigger.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
@Plugin(
examples = {
@Example(
title = "Wait for a SQL query to return results and iterate through rows",
title = "Wait for a SQL query to return results, and then iterate through rows.",
full = true,
code = {
"id: jdbc-trigger",
Expand All @@ -38,13 +38,16 @@
" tasks:",
" - id: return",
" type: io.kestra.core.tasks.debugs.Return",
" format: \"{{json(taskrun.value)}}\"",
" format: \"{{ json(taskrun.value) }}\"",
" value: \"{{ trigger.rows }}\"",
"",
"triggers:",
" - id: watch",
" type: io.kestra.plugin.jdbc.dremio.Trigger",
" interval: \"PT5M\"",
" url: jdbc:dremio:direct=sql.dremio.cloud:443;ssl=true;PROJECT_ID=sampleProjectId;",
" username: $token",
" password: samplePersonalAccessToken",
" sql: \"SELECT * FROM source.database.my_table\""
}
)
Expand Down
5 changes: 3 additions & 2 deletions plugin-jdbc-druid/src/main/java/io/kestra/plugin/jdbc/druid/Query.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,17 +22,18 @@
@Getter
@NoArgsConstructor
@Schema(
title = "Query a Apache Druid server"
title = "Query a Apache Druid database."
)
@Plugin(
examples = {
@Example(
title = "Send a SQL query to Apache Druid database, and fetch a row as output.",
code = {
"url: jdbc:avatica:remote:url=http://localhost:8888/druid/v2/sql/avatica/;transparent_reconnection=true",
"sql: |",
" SELECT *",
" FROM wikiticker",
"fetch: true"
"fetchOne: true"
}
)
}
Expand Down
5 changes: 3 additions & 2 deletions plugin-jdbc-druid/src/main/java/io/kestra/plugin/jdbc/druid/Trigger.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
@Plugin(
examples = {
@Example(
title = "Wait for a sql query to return results and iterate through rows",
title = "Wait for a sql query to return results, and then iterate through rows.",
full = true,
code = {
"id: jdbc-trigger",
Expand All @@ -39,13 +39,14 @@
" tasks:",
" - id: return",
" type: io.kestra.core.tasks.debugs.Return",
" format: \"{{json(taskrun.value)}}\"",
" format: \"{{ json(taskrun.value) }}\"",
" value: \"{{ trigger.rows }}\"",
"",
"triggers:",
" - id: watch",
" type: io.kestra.plugin.jdbc.druid.Trigger",
" interval: \"PT5M\"",
" url: jdbc:avatica:remote:url=http://localhost:8888/druid/v2/sql/avatica/;transparent_reconnection=true",
" sql: \"SELECT * FROM my_table\""
}
)
Expand Down
22 changes: 11 additions & 11 deletions plugin-jdbc-duckdb/src/main/java/io/kestra/plugin/jdbc/duckdb/Query.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,17 +34,17 @@
@Getter
@NoArgsConstructor
@Schema(
title = "Query a local DuckDb"
title = "Query a local DuckDb."
)
@Plugin(
examples = {
@Example(
title = "Execute a query that read a csv and output another one",
title = "Execute a query that reads a csv, and outputs another csv.",
code = {
"url: 'jdbc:duckdb:'",
"timeZoneId: Europe/Paris",
"sql: |-",
" CREATE TABLE new_tbl AS SELECT * FROM read_csv_auto('{{workingDir}}/in.csv', header=True);",
" CREATE TABLE new_tbl AS SELECT * FROM read_csv_auto('{{ workingDir }}/in.csv', header=True);",
"",
" COPY (SELECT id, name FROM new_tbl) TO '{{ outputFiles.out }}' (HEADER, DELIMITER ',');",
"inputFiles:",
Expand All @@ -63,7 +63,7 @@ public class Query extends AbstractJdbcQuery implements RunnableTask<Query.Outpu
@Schema(
title = "Input files to be loaded from DuckDb.",
description = "Describe a files map that will be written and usable by DuckDb. " +
"You can reach files using a `workingDir` variable, example: `SELECT * FROM read_csv_auto('{{workingDir}}/myfile.csv');` "
"You can reach files using a `workingDir` variable, example: `SELECT * FROM read_csv_auto('{{ workingDir }}/myfile.csv');` "
)
@PluginProperty(
additionalProperties = String.class,
Expand All @@ -72,11 +72,11 @@ public class Query extends AbstractJdbcQuery implements RunnableTask<Query.Outpu
protected Object inputFiles;

@Schema(
title = "Output file list that will be uploaded to internal storage",
description = "List of key that will generate temporary files.\n" +
"On the sql query, just can use with special variable named `outputFiles.key`.\n" +
"If you add a files with `[\"first\"]`, you can use the special vars `COPY tbl TO '{[ outputFiles.first }}' (HEADER, DELIMITER ',');`" +
" and you used on others tasks using `{{ outputs.taskId.outputFiles.first }}`"
title = "Output file list that will be uploaded to internal storage.",
description = "List of keys that will generate temporary files.\n" +
"On the SQL query, you can just use a variable named `outputFiles.key` for the corresponding file.\n" +
"If you add a file with `[\"first\"]`, you can use the special vars `COPY tbl TO '{{ outputFiles.first }}' (HEADER, DELIMITER ',');`" +
" and use this file in others tasks using `{{ outputs.taskId.outputFiles.first }}`."
)
@PluginProperty
protected List<String> outputFiles;
Expand All @@ -100,7 +100,7 @@ public void registerDriver() throws SQLException {

@Override
@Schema(
title = "The JDBC URL to connect to the database",
title = "The JDBC URL to connect to the database.",
description = "The default value, `jdbc:duckdb:`, will use a local in-memory database. \nSet this property when connecting to a persisted database instance, for example `jdbc:duckdb:md:my_database?motherduck_token=<my_token>` to connect to [MotherDuck](https://motherduck.com/).",
defaultValue = DEFAULT_URL
)
Expand Down Expand Up @@ -168,7 +168,7 @@ public Query.Output run(RunContext runContext) throws Exception {
@Getter
public static class Output extends AbstractJdbcQuery.Output {
@Schema(
title = "The output files uri in Kestra internal storage"
title = "The output files' URI in Kestra internal storage."
)
@PluginProperty(additionalProperties = URI.class)
private final Map<String, URI> outputFiles;
Expand Down
7 changes: 4 additions & 3 deletions plugin-jdbc-duckdb/src/main/java/io/kestra/plugin/jdbc/duckdb/Trigger.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,12 @@
@Getter
@NoArgsConstructor
@Schema(
title = "Wait for query on a Duckdb database."
title = "Wait for query on a DuckDb database."
)
@Plugin(
examples = {
@Example(
title = "Wait for a sql query to return results and iterate through rows",
title = "Wait for a sql query to return results, and then iterate through rows.",
full = true,
code = {
"id: jdbc-trigger",
Expand All @@ -37,13 +37,14 @@
" tasks:",
" - id: return",
" type: io.kestra.core.tasks.debugs.Return",
" format: \"{{json(taskrun.value)}}\"",
" format: \"{{ json(taskrun.value) }}\"",
" value: \"{{ trigger.rows }}\"",
"",
"triggers:",
" - id: watch",
" type: io.kestra.plugin.jdbc.duckdb.Trigger",
" interval: \"PT5M\"",
" url: 'jdbc:duckdb:'",
" sql: \"SELECT * FROM my_table\""
}
)
Expand Down
8 changes: 4 additions & 4 deletions plugin-jdbc-mysql/src/main/java/io/kestra/plugin/jdbc/mysql/Query.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,19 +33,19 @@
@Plugin(
examples = {
@Example(
title = "Send a sql query to a MySQL Database and fetch a row as outputs",
title = "Send a sql query to a MySQL Database and fetch a row as output.",
code = {
"url: jdbc:mysql://127.0.0.1:56982/",
"url: jdbc:mysql://127.0.0.1:3306/",
"username: mysql_user",
"password: mysql_passwd",
"sql: select * from mysql_types",
"fetchOne: true",
}
),
@Example(
title = "Load a csv file into a MySQL table",
title = "Load a csv file into a MySQL table.",
code = {
"url: jdbc:mysql://127.0.0.1:56982/",
"url: jdbc:mysql://127.0.0.1:3306/",
"username: mysql_user",
"password: mysql_passwd",
"inputFile: \"{{ outputs.taskId.file }}\"",
Expand Down
7 changes: 5 additions & 2 deletions plugin-jdbc-mysql/src/main/java/io/kestra/plugin/jdbc/mysql/Trigger.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
@Plugin(
examples = {
@Example(
title = "Wait for a sql query to return results and iterate through rows",
title = "Wait for a sql query to return results, and then iterate through rows.",
full = true,
code = {
"id: jdbc-trigger",
Expand All @@ -36,13 +36,16 @@
" tasks:",
" - id: return",
" type: io.kestra.core.tasks.debugs.Return",
" format: \"{{json(taskrun.value)}}\"",
" format: \"{{ json(taskrun.value) }}\"",
" value: \"{{ trigger.rows }}\"",
"",
"triggers:",
" - id: watch",
" type: io.kestra.plugin.jdbc.mysql.Trigger",
" interval: \"PT5M\"",
" url: jdbc:mysql://127.0.0.1:3306/",
" username: mysql_user",
" password: mysql_passwd",
" sql: \"SELECT * FROM my_table\""
}
)
Expand Down
10 changes: 5 additions & 5 deletions plugin-jdbc-oracle/src/main/java/io/kestra/plugin/jdbc/oracle/Query.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,28 +24,28 @@
@Getter
@NoArgsConstructor
@Schema(
title = "Query a Oracle server"
title = "Query an Oracle database."
)
@Plugin(
examples = {
@Example(
full = true,
title = "Execute a query and fetch results on another task to update another table",
title = "Execute a query and fetch results on another task to update another table.",
code = {
"tasks:",
"- id: select",
" type: io.kestra.plugin.jdbc.oracle.Query",
" url: jdbc:oracle:thin:@localhost:49161:XE",
" username: oracle",
" username: oracle_user",
" password: oracle_passwd",
" sql: select * from source",
" fetch: true",
"- id: generate-update",
" type: io.kestra.plugin.jdbc.oracle.Query",
" url: jdbc:oracle:thin:@localhost:49161:XE",
" username: oracle",
" username: oracle_user",
" password: oracle_passwd",
" sql: \"{% for row in outputs.update.rows %} INSERT INTO destination (year_month, store_code, update_date) values ({{row.year_month}}, {{row.store_code}}, TO_DATE('{{row.date}}', 'MONTH DD, YYYY') ); {% endfor %}\""}
" sql: \"{% for row in outputs.select.rows %} INSERT INTO destination (year_month, store_code, update_date) values ({{ row.year_month }}, {{ row.store_code }}, TO_DATE('{{ row.date }}', 'MONTH DD, YYYY') ); {% endfor %}\""}
)
}
)
Expand Down
Loading

0 comments on commit 85ea919

Please sign in to comment.