Skip to content

Commit

Permalink
fix(docs): add doc corrections for multiple jdbc plugins (#206)
Browse files Browse the repository at this point in the history
* fix(docs): add doc corrections for multiple jdbc plugins

* Apply suggestions from code review

---------

Co-authored-by: Shruti Mantri <smantri@moveworks.ai>
Co-authored-by: Loïc Mathieu <loikeseke@gmail.com>
  • Loading branch information
3 people authored Jan 2, 2024
1 parent 85ea919 commit 99ec66b
Show file tree
Hide file tree
Showing 25 changed files with 117 additions and 102 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
@Plugin(
examples = {
@Example(
title = "Wait for a sql query to return results, and then iterate through rows.",
title = "Wait for a SQL query to return results, and then iterate through rows.",
full = true,
code = {
"id: jdbc-trigger",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
@Plugin(
examples = {
@Example(
title = "Wait for a sql query to return results, and then iterate through rows.",
title = "Wait for a SQL query to return results, and then iterate through rows.",
full = true,
code = {
"id: jdbc-trigger",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
@Plugin(
examples = {
@Example(
title = "Wait for a sql query to return results, and then iterate through rows.",
title = "Wait for a SQL query to return results, and then iterate through rows.",
full = true,
code = {
"id: jdbc-trigger",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
@Plugin(
examples = {
@Example(
title = "Wait for a sql query to return results, and then iterate through rows.",
title = "Wait for a SQL query to return results, and then iterate through rows.",
full = true,
code = {
"id: jdbc-trigger",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
@Plugin(
examples = {
@Example(
title = "Wait for a sql query to return results, and then iterate through rows.",
title = "Wait for a SQL query to return results, and then iterate through rows.",
full = true,
code = {
"id: jdbc-trigger",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
@Getter
@NoArgsConstructor
@Schema(
title = "Query a Apache Pinot server"
title = "Query a Apache Pinot server."
)
@Plugin(
examples = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
@Plugin(
examples = {
@Example(
title = "Wait for a sql query to return results and iterate through rows",
title = "Wait for a SQL query to return results, and then iterate through rows.",
full = true,
code = {
"id: jdbc-trigger",
Expand All @@ -39,13 +39,14 @@
" tasks:",
" - id: return",
" type: io.kestra.core.tasks.debugs.Return",
" format: \"{{json(taskrun.value)}}\"",
" format: \"{{ json(taskrun.value) }}\"",
" value: \"{{ trigger.rows }}\"",
"",
"triggers:",
" - id: watch",
" type: io.kestra.plugin.jdbc.pinot.Trigger",
" interval: \"PT5M\"",
" url: jdbc:pinot://localhost:9000",
" sql: \"SELECT * FROM my_table\""
}
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,51 +39,51 @@ public abstract class AbstractCopy extends Task implements PostgresConnectionInt
protected String table;

@Schema(
title = "An optional list of columns to be copied",
title = "An optional list of columns to be copied.",
description = "If no column list is specified, all columns of the table will be copied."
)
@PluginProperty(dynamic = false)
protected List<String> columns;

@Schema(
title = "Selects the data format to be read or written"
title = "Selects the data format to be read or written."
)
@PluginProperty(dynamic = false)
@Builder.Default
protected Format format = Format.TEXT;

@Schema(
title = "Specifies copying the OID for each row",
description = "An error is raised if OIDS is specified for a table that does not have OIDs, or in the case of copying a query."
title = "Specifies copying the OID for each row.",
description = "An error is raised if OIDs is specified for a table that does not have OIDs, or in the case of copying a query."
)
@PluginProperty(dynamic = false)
protected Boolean oids;

@Schema(
title = "Requests copying the data with rows already frozen, just as they would be after running the VACUUM FREEZE command",
description = "This is intended as a performance option for initial data loading. Rows will be frozen only if the table being loaded has been created or truncated in the current subtransaction, there are no cursors open and there are no older snapshots held by this transaction. It is currently not possible to perform a COPY FREEZE on a partitioned table.\n" +
title = "Requests copying the data with rows already frozen, just as they would be after running the VACUUM FREEZE command.",
description = "This is intended as a performance option for initial data loading. Rows will be frozen only if the table being loaded has been created or truncated in the current sub-transaction, there are no cursors open and there are no older snapshots held by this transaction. It is currently not possible to perform a COPY FREEZE on a partitioned table.\n" +
"\n" +
"Note that all other sessions will immediately be able to see the data once it has been successfully loaded. This violates the normal rules of MVCC visibility and users specifying should be aware of the potential problems this might cause."
)
@PluginProperty(dynamic = false)
protected Boolean freeze;

@Schema(
title = "Specifies the character that separates columns within each row (line) of the file",
description = "The default is a tab character in text format, a comma in CSV format. This must be a single one-byte character. This option is not allowed when using binary"
title = "Specifies the character that separates columns within each row (line) of the file.",
description = "The default is a tab character in text format, a comma in CSV format. This must be a single one-byte character. This option is not allowed when using binary."
)
@PluginProperty(dynamic = false)
protected Character delimiter;

@Schema(
title = "Specifies the string that represents a null value",
title = "Specifies the string that represents a null value.",
description = "The default is \\N (backslash-N) in text format, and an unquoted empty string in CSV format. You might prefer an empty string even in text format for cases where you don't want to distinguish nulls from empty strings. This option is not allowed when using binary format."
)
@PluginProperty(dynamic = false)
protected String nullString;

@Schema(
title = "Specifies that the file contains a header line with the names of each column in the file",
title = "Specifies that the file contains a header line with the names of each column in the file.",
description = "On output, the first line contains the column names from the table, and on input, the first line is ignored. This option is allowed only when using CSV."
)
@PluginProperty(dynamic = false)
Expand All @@ -104,28 +104,28 @@ public abstract class AbstractCopy extends Task implements PostgresConnectionInt
protected Character escape;

@Schema(
title = "Forces quoting to be used for all non-NULL values in each specified column",
title = "Forces quoting to be used for all non-NULL values in each specified column.",
description = "NULL output is never quoted. If * is specified, non-NULL values will be quoted in all columns. This option is allowed only in COPY TO, and only when using CSV format."
)
@PluginProperty(dynamic = false)
protected List<String> forceQuote;

@Schema(
title = "Do not match the specified columns' values against the null string",
title = "Do not match the specified columns' values against the null string.",
description = "In the default case where the null string is empty, this means that empty values will be read as zero-length strings rather than nulls, even when they are not quoted. This option is allowed only in COPY FROM, and only when using CSV format."
)
@PluginProperty(dynamic = false)
protected List<String> forceNotNull;

@Schema(
title = "Match the specified columns' values against the null string, even if it has been quoted, and if a match is found set the value to NULL",
title = "Match the specified columns' values against the null string, even if it has been quoted, and if a match is found set the value to NULL.",
description = "In the default case where the null string is empty, this converts a quoted empty string into NULL. This option is allowed only in COPY FROM, and only when using CSV format."
)
@PluginProperty(dynamic = false)
protected List<String> forceNull;

@Schema(
title = "Specifies that the file is encoded in the encoding_name",
title = "Specifies that the file is encoded in the encoding_name.",
description = "If this option is omitted, the current client encoding is used. See the Notes below for more details."
)
@PluginProperty(dynamic = false)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,19 +23,19 @@
@Getter
@NoArgsConstructor
@Schema(
title = "Execute a batch query to a PostgresSQL server"
title = "Execute a batch query to a PostgreSQL server."
)
@Plugin(
examples = {
@Example(
title = "Fetch rows from a table and bulk insert to another one",
title = "Fetch rows from a table, and bulk insert them to another one.",
full = true,
code = {
"tasks:",
" - id: query",
" type: io.kestra.plugin.jdbc.postgresql.Query",
" url: jdbc:postgresql://dev:56982/",
" username: postgres",
" username: pg_user",
" password: pg_passwd",
" sql: |",
" SELECT *",
Expand All @@ -46,7 +46,7 @@
" type: io.kestra.plugin.jdbc.postgresql.Batch",
" from: \"{{ outputs.query.uri }}\"",
" url: jdbc:postgresql://prod:56982/",
" username: postgres",
" username: pg_user",
" password: pg_passwd",
" sql: |",
" insert into xref values( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,16 @@
@Getter
@NoArgsConstructor
@Schema(
title = "Copy a PostgreSQL table or a query to a file",
description = "Moves data between PostgreSQL tables or query to a file"
title = "Copy a PostgreSQL table or a query to a file.",
description = "Moves data between PostgreSQL tables or query to a file."
)
@Plugin(
examples = {
@Example(
title = "Load csv or tsv into a postgres table",
title = "Load CSV or TSV into a PostgreSQL table.",
code = {
"url: jdbc:postgresql://127.0.0.1:56982/",
"username: postgres",
"username: pg_user",
"password: pg_passwd",
"format: CSV",
"from: {{ outputs.export.uri }}",
Expand All @@ -48,7 +48,7 @@
public class CopyIn extends AbstractCopy implements RunnableTask<CopyIn.Output>, PostgresConnectionInterface {
@NotNull
@io.swagger.v3.oas.annotations.media.Schema(
title = "Source file URI"
title = "Source file URI."
)
@PluginProperty(dynamic = true)
private String from;
Expand Down Expand Up @@ -83,7 +83,7 @@ public Output run(RunContext runContext) throws Exception {
@Getter
public static class Output implements io.kestra.core.models.tasks.Output {
@Schema(
title = "The rows count from this `COPY`"
title = "The rows count from this `COPY`."
)
private final Long rowCount;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,15 @@
@Getter
@NoArgsConstructor
@Schema(
title = "Copy tabular data from a Postgres table to a file"
title = "Copy tabular data from a Postgres table to a file."
)
@Plugin(
examples = {
@Example(
title = "Export a Postgres table or query to a CSV or TSV file",
title = "Export a Postgres table or query to a CSV or TSV file.",
code = {
"url: jdbc:postgresql://127.0.0.1:56982/",
"username: postgres",
"username: pg_user",
"password: pg_passwd",
"format: CSV",
"sql: SELECT 1 AS int, 't'::bool AS bool UNION SELECT 2 AS int, 'f'::bool AS bool",
Expand Down Expand Up @@ -84,12 +84,12 @@ public Output run(RunContext runContext) throws Exception {
@Getter
public static class Output implements io.kestra.core.models.tasks.Output {
@Schema(
title = "The url of the result file on kestra storage"
title = "The URI of the result file on Kestra's internal storage."
)
private final URI uri;

@Schema(
title = "The rows count from this `COPY`"
title = "The rows count from this `COPY`."
)
private final Long rowCount;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,40 +6,40 @@

public interface PostgresConnectionInterface extends JdbcConnectionInterface {
@Schema(
title = "Is the connection ssl"
title = "Is the connection SSL?"
)
@PluginProperty(dynamic = false)
Boolean getSsl();

@Schema(
title = "The ssl mode"
title = "The SSL mode."
)
@PluginProperty(dynamic = false)
SslMode getSslMode();

@Schema(
title = "The ssl root cert",
title = "The SSL root cert.",
description = "Must be a PEM encoded certificate"
)
@PluginProperty(dynamic = true)
String getSslRootCert();

@Schema(
title = "The ssl cert",
title = "The SSL cert.",
description = "Must be a PEM encoded certificate"
)
@PluginProperty(dynamic = true)
String getSslCert();

@Schema(
title = "The ssl key",
title = "The SSL key.",
description = "Must be a PEM encoded key"
)
@PluginProperty(dynamic = true)
String getSslKey();

@Schema(
title = "The ssl key password"
title = "The SSL key password."
)
@PluginProperty(dynamic = true)
String getSslKeyPassword();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,36 +17,37 @@
import java.util.Properties;



@SuperBuilder
@ToString
@EqualsAndHashCode
@Getter
@NoArgsConstructor
@Schema(
title = "Query a PostgresSQL server"
title = "Query a PostgreSQL server."
)
@Plugin(
examples = {
@Example(
full = true,
title = "Execute a query and fetch results on another task to update another table",
code = {
"tasks:",
"- id: update",
" type: io.kestra.plugin.jdbc.postgresql.Query",
" url: jdbc:postgresql://127.0.0.1:56982/",
" username: postgres",
" password: pg_passwd",
" sql: select concert_id, available, a, b, c, d, play_time, library_record, floatn_test, double_test, real_test, numeric_test, date_type, time_type, timez_type, timestamp_type, timestampz_type, interval_type, pay_by_quarter, schedule, json_type, blob_type from pgsql_types",
" fetch: true",
"- id: use-fetched-data",
" type: io.kestra.plugin.jdbc.postgresql.Query",
" url: jdbc:postgresql://127.0.0.1:56982/",
" username: postgres",
" password: pg_passwd",
" sql: \"{% for row in outputs.update.rows %} INSERT INTO pl_store_distribute (year_month,store_code, update_date) values ({{row.play_time}}, {{row.concert_id}}, TO_TIMESTAMP('{{row.timestamp_type}}', 'YYYY-MM-DDTHH:MI:SS.US') ); {% endfor %}\""}
)
}
examples = {
@Example(
full = true,
title = "Execute a query and fetch results in a task, and update another table with fetched results in a different task.",
code = {
"tasks:",
"- id: fetch",
" type: io.kestra.plugin.jdbc.postgresql.Query",
" url: jdbc:postgresql://127.0.0.1:56982/",
" username: pg_user",
" password: pg_passwd",
" sql: select concert_id, available, a, b, c, d, play_time, library_record, floatn_test, double_test, real_test, numeric_test, date_type, time_type, timez_type, timestamp_type, timestampz_type, interval_type, pay_by_quarter, schedule, json_type, blob_type from pgsql_types",
" fetch: true",
"- id: use-fetched-data",
" type: io.kestra.plugin.jdbc.postgresql.Query",
" url: jdbc:postgresql://127.0.0.1:56982/",
" username: pg_user",
" password: pg_passwd",
" sql: \"{% for row in outputs.fetch.rows %} INSERT INTO pl_store_distribute (year_month,store_code, update_date) values ({{row.play_time}}, {{row.concert_id}}, TO_TIMESTAMP('{{row.timestamp_type}}', 'YYYY-MM-DDTHH:MI:SS.US') ); {% endfor %}\""}
)
}
)
public class Query extends AbstractJdbcQuery implements RunnableTask<AbstractJdbcQuery.Output>, PostgresConnectionInterface, AutoCommitInterface {
protected final Boolean autoCommit = true;
Expand Down
Loading

0 comments on commit 99ec66b

Please sign in to comment.