diff --git a/examples/src/main/java/org/apache/spark/examples/sql/hive/JavaSparkHiveExample.java b/examples/src/main/java/org/apache/spark/examples/sql/hive/JavaSparkHiveExample.java index 575a463e8725..7639fb8d84fb 100644 --- a/examples/src/main/java/org/apache/spark/examples/sql/hive/JavaSparkHiveExample.java +++ b/examples/src/main/java/org/apache/spark/examples/sql/hive/JavaSparkHiveExample.java @@ -124,6 +124,13 @@ public static void main(String[] args) { // ... // $example off:spark_hive$ + // Hive serde's are also supported with serde properties. + String sqlQuery = "CREATE TABLE src_serde(key decimal(38,18), value int) USING hive" + + " OPTIONS (SERDE 'org.apache.hadoop.hive.serde2.RegexSerDe', input.regex '([^ ]*) ([^ ]*)')"; + spark.sql(sqlQuery); + spark.sql("LOAD DATA LOCAL INPATH 'examples/src/main/resources/kv7.txt' INTO TABLE src_serde"); + // Executed the data with the specified serde properties. + spark.sql("SELECT key, value FROM src_serde ORDER BY key, value").show(); spark.stop(); } } diff --git a/examples/src/main/python/sql/hive.py b/examples/src/main/python/sql/hive.py index 1f83a6fb48b9..b437c89ae26c 100644 --- a/examples/src/main/python/sql/hive.py +++ b/examples/src/main/python/sql/hive.py @@ -92,5 +92,13 @@ # | 5| val_5| 5| val_5| # ... # $example off:spark_hive$ + + # Hive serde's are also supported with serde properties. + sqlQuery = """CREATE TABLE src_serde(key decimal(38,18), value int) USING hive + OPTIONS (SERDE 'org.apache.hadoop.hive.serde2.RegexSerDe', input.regex '([^ ]*) ([^ ]*)')""" + spark.sql(sqlQuery) + spark.sql("LOAD DATA LOCAL INPATH 'examples/src/main/resources/kv7.txt' INTO TABLE src_serde") + # Executed the data with the specified serde properties. + spark.sql("SELECT key, value FROM src_serde ORDER BY key, value").show() spark.stop() diff --git a/examples/src/main/resources/kv7.txt b/examples/src/main/resources/kv7.txt new file mode 100644 index 000000000000..b83c46099bbd --- /dev/null +++ b/examples/src/main/resources/kv7.txt @@ -0,0 +1,38 @@ +-4400 4400 +1E+99 0 +1E-99 0 +0 0 +100 100 +10 10 +1 1 +0.1 0 +0.01 0 +200 200 +20 20 +2 2 +0 0 +0.2 0 +0.02 0 +0.3 0 +0.33 0 +0.333 0 +-0.3 0 +-0.33 0 +-0.333 0 +1.0 1 +2 2 +3.14 3 +-1.12 -1 +-1.12 -1 +-1.122 -11 +1.12 1 +1.122 1 +124.00 124 +125.2 125 +-1255.49 -1255 +3.14 3 +3.14 3 +3.140 4 +0.9999999999999999999999999 1 +-1234567890.1234567890 -1234567890 +1234567890.1234567800 1234567890 diff --git a/examples/src/main/scala/org/apache/spark/examples/sql/hive/SparkHiveExample.scala b/examples/src/main/scala/org/apache/spark/examples/sql/hive/SparkHiveExample.scala index e5f75d53edc8..4f94a149bfbc 100644 --- a/examples/src/main/scala/org/apache/spark/examples/sql/hive/SparkHiveExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/sql/hive/SparkHiveExample.scala @@ -104,6 +104,13 @@ object SparkHiveExample { // ... // $example off:spark_hive$ + // Hive serde's are also supported with serde properties. + val sqlQuery = "CREATE TABLE src_serde(key decimal(38,18), value int) USING hive" + + " OPTIONS (SERDE 'org.apache.hadoop.hive.serde2.RegexSerDe', input.regex '([^ ]*) ([^ ]*)')" + sql(sqlQuery) + sql("LOAD DATA LOCAL INPATH 'examples/src/main/resources/kv7.txt' INTO TABLE src_serde") + // Executed the data with the specified serde properties. + sql("SELECT key, value FROM src_serde ORDER BY key, value").show() spark.stop() } }