diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
index 93bf74d06b71..d29d90ce4045 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
@@ -269,14 +269,15 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
*
`allowBackslashEscapingAnyCharacter` (default `false`): allows accepting quoting of all
* character using backslash quoting mechanism
* `mode` (default `PERMISSIVE`): allows a mode for dealing with corrupt records
- * during parsing.
- *
- * - - `PERMISSIVE` : sets other fields to `null` when it meets a corrupted record, and puts
- * the malformed string into a new field configured by `columnNameOfCorruptRecord`. When
- * a schema is set by user, it sets `null` for extra fields.
- * - - `DROPMALFORMED` : ignores the whole corrupted records.
- * - - `FAILFAST` : throws an exception when it meets corrupted records.
- *
+ * during parsing.
+ *
+ * - `PERMISSIVE` : sets other fields to `null` when it meets a corrupted record, and puts
+ * the malformed string into a new field configured by `columnNameOfCorruptRecord`. When
+ * a schema is set by user, it sets `null` for extra fields.
+ * - `DROPMALFORMED` : ignores the whole corrupted records.
+ * - `FAILFAST` : throws an exception when it meets corrupted records.
+ *
+ *
* `columnNameOfCorruptRecord` (default is the value specified in
* `spark.sql.columnNameOfCorruptRecord`): allows renaming the new field having malformed string
* created by `PERMISSIVE` mode. This overrides `spark.sql.columnNameOfCorruptRecord`.
@@ -395,13 +396,14 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
* `maxMalformedLogPerPartition` (default `10`): sets the maximum number of malformed rows
* Spark will log for each partition. Malformed records beyond this number will be ignored.
* `mode` (default `PERMISSIVE`): allows a mode for dealing with corrupt records
- * during parsing.
- *
- * - - `PERMISSIVE` : sets other fields to `null` when it meets a corrupted record. When
- * a schema is set by user, it sets `null` for extra fields.
- * - - `DROPMALFORMED` : ignores the whole corrupted records.
- * - - `FAILFAST` : throws an exception when it meets corrupted records.
- *
+ * during parsing.
+ *
+ * - `PERMISSIVE` : sets other fields to `null` when it meets a corrupted record. When
+ * a schema is set by user, it sets `null` for extra fields.
+ * - `DROPMALFORMED` : ignores the whole corrupted records.
+ * - `FAILFAST` : throws an exception when it meets corrupted records.
+ *
+ *
*
* @since 2.0.0
*/
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
index c05c7a655160..e137f076a0ca 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
@@ -397,7 +397,9 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
* your external database systems.
*
* You can set the following JDBC-specific option(s) for storing JDBC:
+ *
* - `truncate` (default `false`): use `TRUNCATE TABLE` instead of `DROP TABLE`.
+ *
*
* In case of failures, users should turn off `truncate` option to use `DROP TABLE` again. Also,
* due to the different behavior of `TRUNCATE TABLE` among DBMS, it's not always safe to use this.
@@ -486,6 +488,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
* }}}
*
* You can set the following JSON-specific option(s) for writing JSON files:
+ *
* - `compression` (default `null`): compression codec to use when saving to file. This can be
* one of the known case-insensitive shorten names (`none`, `bzip2`, `gzip`, `lz4`,
* `snappy` and `deflate`).
@@ -495,6 +498,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
* - `timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSZZ`): sets the string that
* indicates a timestamp format. Custom date formats follow the formats at
* `java.text.SimpleDateFormat`. This applies to timestamp type.
+ *
*
* @since 1.4.0
*/
@@ -510,10 +514,12 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
* }}}
*
* You can set the following Parquet-specific option(s) for writing Parquet files:
+ *
* - `compression` (default is the value specified in `spark.sql.parquet.compression.codec`):
* compression codec to use when saving to file. This can be one of the known case-insensitive
* shorten names(none, `snappy`, `gzip`, and `lzo`). This will override
* `spark.sql.parquet.compression.codec`.
+ *
*
* @since 1.4.0
*/
@@ -529,9 +535,11 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
* }}}
*
* You can set the following ORC-specific option(s) for writing ORC files:
+ *
* - `compression` (default `snappy`): compression codec to use when saving to file. This can be
* one of the known case-insensitive shorten names(`none`, `snappy`, `zlib`, and `lzo`).
* This will override `orc.compress`.
+ *
*
* @since 1.5.0
* @note Currently, this method can only be used after enabling Hive support
@@ -553,9 +561,11 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
* }}}
*
* You can set the following option(s) for writing text files:
+ *
* - `compression` (default `null`): compression codec to use when saving to file. This can be
* one of the known case-insensitive shorten names (`none`, `bzip2`, `gzip`, `lz4`,
* `snappy` and `deflate`).
+ *
*
* @since 1.6.0
*/
@@ -571,6 +581,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
* }}}
*
* You can set the following CSV-specific option(s) for writing CSV files:
+ *
* - `sep` (default `,`): sets the single character as a separator for each
* field and value.
* - `quote` (default `"`): sets the single character used for escaping quoted values where
@@ -593,6 +604,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
*
- `timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSZZ`): sets the string that
* indicates a timestamp format. Custom date formats follow the formats at
* `java.text.SimpleDateFormat`. This applies to timestamp type.
+ *
*
* @since 2.0.0
*/
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
index 3ad1125229c9..c25f71af7362 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
@@ -161,6 +161,7 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
* schema in advance, use the version that specifies the schema to avoid the extra scan.
*
* You can set the following JSON-specific options to deal with non-standard JSON files:
+ *
* - `maxFilesPerTrigger` (default: no max limit): sets the maximum number of new files to be
* considered in every trigger.
* - `primitivesAsString` (default `false`): infers all primitive values as a string type
@@ -175,14 +176,15 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
* - `allowBackslashEscapingAnyCharacter` (default `false`): allows accepting quoting of all
* character using backslash quoting mechanism
* - `mode` (default `PERMISSIVE`): allows a mode for dealing with corrupt records
- * during parsing.
- *
- * - `PERMISSIVE` : sets other fields to `null` when it meets a corrupted record, and puts the
- * malformed string into a new field configured by `columnNameOfCorruptRecord`. When
- * a schema is set by user, it sets `null` for extra fields.
- * - `DROPMALFORMED` : ignores the whole corrupted records.
- * - `FAILFAST` : throws an exception when it meets corrupted records.
- *
+ * during parsing.
+ *
+ * - `PERMISSIVE` : sets other fields to `null` when it meets a corrupted record, and puts
+ * the malformed string into a new field configured by `columnNameOfCorruptRecord`. When
+ * a schema is set by user, it sets `null` for extra fields.
+ * - `DROPMALFORMED` : ignores the whole corrupted records.
+ * - `FAILFAST` : throws an exception when it meets corrupted records.
+ *
+ *
* - `columnNameOfCorruptRecord` (default is the value specified in
* `spark.sql.columnNameOfCorruptRecord`): allows renaming the new field having malformed string
* created by `PERMISSIVE` mode. This overrides `spark.sql.columnNameOfCorruptRecord`.
@@ -192,6 +194,7 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
* - `timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSZZ`): sets the string that
* indicates a timestamp format. Custom date formats follow the formats at
* `java.text.SimpleDateFormat`. This applies to timestamp type.
+ *
*
* @since 2.0.0
*/
@@ -207,6 +210,7 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
* specify the schema explicitly using [[schema]].
*
* You can set the following CSV-specific options to deal with CSV files:
+ *
* - `maxFilesPerTrigger` (default: no max limit): sets the maximum number of new files to be
* considered in every trigger.
* - `sep` (default `,`): sets the single character as a separator for each
@@ -245,12 +249,14 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
*
- `maxCharsPerColumn` (default `1000000`): defines the maximum number of characters allowed
* for any given value being read.
* - `mode` (default `PERMISSIVE`): allows a mode for dealing with corrupt records
- * during parsing.
- *
- * - `PERMISSIVE` : sets other fields to `null` when it meets a corrupted record. When
- * a schema is set by user, it sets `null` for extra fields.
- * - `DROPMALFORMED` : ignores the whole corrupted records.
- * - `FAILFAST` : throws an exception when it meets corrupted records.
+ * during parsing.
+ *
+ * - `PERMISSIVE` : sets other fields to `null` when it meets a corrupted record. When
+ * a schema is set by user, it sets `null` for extra fields.
+ * - `DROPMALFORMED` : ignores the whole corrupted records.
+ * - `FAILFAST` : throws an exception when it meets corrupted records.
+ *
+ *
*
*
* @since 2.0.0
@@ -263,12 +269,14 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
* Loads a Parquet file stream, returning the result as a [[DataFrame]].
*
* You can set the following Parquet-specific option(s) for reading Parquet files:
+ *
* - `maxFilesPerTrigger` (default: no max limit): sets the maximum number of new files to be
* considered in every trigger.
* - `mergeSchema` (default is the value specified in `spark.sql.parquet.mergeSchema`): sets
* whether we should merge schemas collected from all
* Parquet part-files. This will override
* `spark.sql.parquet.mergeSchema`.
+ *
*
* @since 2.0.0
*/
@@ -292,8 +300,10 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
* }}}
*
* You can set the following text-specific options to deal with text files:
+ *
* - `maxFilesPerTrigger` (default: no max limit): sets the maximum number of new files to be
* considered in every trigger.
+ *
*
* @since 2.0.0
*/