Skip to content

Commit 73bb605

Browse files
[SPARK-27168][SQL][TEST] Add docker integration test for MsSql server
## What changes were proposed in this pull request? This PR aims to add a JDBC integration test for MsSql server. ## How was this patch tested? ``` ./build/mvn clean install -DskipTests ./build/mvn test -Pdocker-integration-tests -pl :spark-docker-integration-tests_2.12 \ -Dtest=none -DwildcardSuites=org.apache.spark.sql.jdbc.MsSqlServerIntegrationSuite ``` Closes #24099 from lipzhu/SPARK-27168. Lead-authored-by: Zhu, Lipeng <lipzhu@ebay.com> Co-authored-by: Dongjoon Hyun <dhyun@apple.com> Co-authored-by: Lipeng Zhu <lipzhu@icloud.com> Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
1 parent 4336d1c commit 73bb605

File tree

2 files changed

+211
-0
lines changed

2 files changed

+211
-0
lines changed

external/docker-integration-tests/pom.xml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,5 +150,11 @@
150150
<version>10.5.0.5</version>
151151
<type>jar</type>
152152
</dependency>
153+
<dependency>
154+
<groupId>com.microsoft.sqlserver</groupId>
155+
<artifactId>mssql-jdbc</artifactId>
156+
<version>7.2.1.jre8</version>
157+
<scope>test</scope>
158+
</dependency>
153159
</dependencies>
154160
</project>
Lines changed: 205 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.spark.sql.jdbc
19+
20+
import java.math.BigDecimal
21+
import java.sql.{Connection, Date, Timestamp}
22+
import java.util.Properties
23+
24+
import org.apache.spark.tags.DockerTest
25+
26+
@DockerTest
27+
class MsSqlServerIntegrationSuite extends DockerJDBCIntegrationSuite {
28+
override val db = new DatabaseOnDocker {
29+
override val imageName = "mcr.microsoft.com/mssql/server:2017-GA-ubuntu"
30+
override val env = Map(
31+
"SA_PASSWORD" -> "Sapass123",
32+
"ACCEPT_EULA" -> "Y"
33+
)
34+
override val usesIpc = false
35+
override val jdbcPort: Int = 1433
36+
37+
override def getJdbcUrl(ip: String, port: Int): String =
38+
s"jdbc:sqlserver://$ip:$port;user=sa;password=Sapass123;"
39+
40+
override def getStartupProcessName: Option[String] = None
41+
}
42+
43+
override def dataPreparation(conn: Connection): Unit = {
44+
conn.prepareStatement("CREATE TABLE tbl (x INT, y VARCHAR (50))").executeUpdate()
45+
conn.prepareStatement("INSERT INTO tbl VALUES (42,'fred')").executeUpdate()
46+
conn.prepareStatement("INSERT INTO tbl VALUES (17,'dave')").executeUpdate()
47+
48+
conn.prepareStatement(
49+
"""
50+
|CREATE TABLE numbers (
51+
|a BIT,
52+
|b TINYINT, c SMALLINT, d INT, e BIGINT,
53+
|f FLOAT, f1 FLOAT(24),
54+
|g REAL,
55+
|h DECIMAL(5,2), i NUMERIC(10,5),
56+
|j MONEY, k SMALLMONEY)
57+
""".stripMargin).executeUpdate()
58+
conn.prepareStatement(
59+
"""
60+
|INSERT INTO numbers VALUES (
61+
|0,
62+
|255, 32767, 2147483647, 9223372036854775807,
63+
|123456789012345.123456789012345, 123456789012345.123456789012345,
64+
|123456789012345.123456789012345,
65+
|123, 12345.12,
66+
|922337203685477.58, 214748.3647)
67+
""".stripMargin).executeUpdate()
68+
69+
conn.prepareStatement(
70+
"""
71+
|CREATE TABLE dates (
72+
|a DATE, b DATETIME, c DATETIME2,
73+
|d DATETIMEOFFSET, e SMALLDATETIME,
74+
|f TIME)
75+
""".stripMargin).executeUpdate()
76+
conn.prepareStatement(
77+
"""
78+
|INSERT INTO dates VALUES (
79+
|'1991-11-09', '1999-01-01 13:23:35', '9999-12-31 23:59:59',
80+
|'1901-05-09 23:59:59 +14:00', '1996-01-01 23:23:45',
81+
|'13:31:24')
82+
""".stripMargin).executeUpdate()
83+
84+
conn.prepareStatement(
85+
"""
86+
|CREATE TABLE strings (
87+
|a CHAR(10), b VARCHAR(10),
88+
|c NCHAR(10), d NVARCHAR(10),
89+
|e BINARY(4), f VARBINARY(4),
90+
|g TEXT, h NTEXT,
91+
|i IMAGE)
92+
""".stripMargin).executeUpdate()
93+
conn.prepareStatement(
94+
"""
95+
|INSERT INTO strings VALUES (
96+
|'the', 'quick',
97+
|'brown', 'fox',
98+
|123456, 123456,
99+
|'the', 'lazy',
100+
|'dog')
101+
""".stripMargin).executeUpdate()
102+
}
103+
104+
test("Basic test") {
105+
val df = spark.read.jdbc(jdbcUrl, "tbl", new Properties)
106+
val rows = df.collect()
107+
assert(rows.length == 2)
108+
val types = rows(0).toSeq.map(x => x.getClass.toString)
109+
assert(types.length == 2)
110+
assert(types(0).equals("class java.lang.Integer"))
111+
assert(types(1).equals("class java.lang.String"))
112+
}
113+
114+
test("Numeric types") {
115+
val df = spark.read.jdbc(jdbcUrl, "numbers", new Properties)
116+
val rows = df.collect()
117+
assert(rows.length == 1)
118+
val row = rows(0)
119+
val types = row.toSeq.map(x => x.getClass.toString)
120+
assert(types.length == 12)
121+
assert(types(0).equals("class java.lang.Boolean"))
122+
assert(types(1).equals("class java.lang.Integer"))
123+
assert(types(2).equals("class java.lang.Integer"))
124+
assert(types(3).equals("class java.lang.Integer"))
125+
assert(types(4).equals("class java.lang.Long"))
126+
assert(types(5).equals("class java.lang.Double"))
127+
assert(types(6).equals("class java.lang.Double"))
128+
assert(types(7).equals("class java.lang.Double"))
129+
assert(types(8).equals("class java.math.BigDecimal"))
130+
assert(types(9).equals("class java.math.BigDecimal"))
131+
assert(types(10).equals("class java.math.BigDecimal"))
132+
assert(types(11).equals("class java.math.BigDecimal"))
133+
assert(row.getBoolean(0) == false)
134+
assert(row.getInt(1) == 255)
135+
assert(row.getInt(2) == 32767)
136+
assert(row.getInt(3) == 2147483647)
137+
assert(row.getLong(4) == 9223372036854775807L)
138+
assert(row.getDouble(5) == 1.2345678901234512E14) // float = float(53) has 15-digits precision
139+
assert(row.getDouble(6) == 1.23456788103168E14) // float(24) has 7-digits precision
140+
assert(row.getDouble(7) == 1.23456788103168E14) // real = float(24)
141+
assert(row.getAs[BigDecimal](8).equals(new BigDecimal("123.00")))
142+
assert(row.getAs[BigDecimal](9).equals(new BigDecimal("12345.12000")))
143+
assert(row.getAs[BigDecimal](10).equals(new BigDecimal("922337203685477.5800")))
144+
assert(row.getAs[BigDecimal](11).equals(new BigDecimal("214748.3647")))
145+
}
146+
147+
test("Date types") {
148+
val df = spark.read.jdbc(jdbcUrl, "dates", new Properties)
149+
val rows = df.collect()
150+
assert(rows.length == 1)
151+
val row = rows(0)
152+
val types = row.toSeq.map(x => x.getClass.toString)
153+
assert(types.length == 6)
154+
assert(types(0).equals("class java.sql.Date"))
155+
assert(types(1).equals("class java.sql.Timestamp"))
156+
assert(types(2).equals("class java.sql.Timestamp"))
157+
assert(types(3).equals("class java.lang.String"))
158+
assert(types(4).equals("class java.sql.Timestamp"))
159+
assert(types(5).equals("class java.sql.Timestamp"))
160+
assert(row.getAs[Date](0).equals(Date.valueOf("1991-11-09")))
161+
assert(row.getAs[Timestamp](1).equals(Timestamp.valueOf("1999-01-01 13:23:35.0")))
162+
assert(row.getAs[Timestamp](2).equals(Timestamp.valueOf("9999-12-31 23:59:59.0")))
163+
assert(row.getString(3).equals("1901-05-09 23:59:59.0000000 +14:00"))
164+
assert(row.getAs[Timestamp](4).equals(Timestamp.valueOf("1996-01-01 23:24:00.0")))
165+
assert(row.getAs[Timestamp](5).equals(Timestamp.valueOf("1900-01-01 13:31:24.0")))
166+
}
167+
168+
test("String types") {
169+
val df = spark.read.jdbc(jdbcUrl, "strings", new Properties)
170+
val rows = df.collect()
171+
assert(rows.length == 1)
172+
val row = rows(0)
173+
val types = row.toSeq.map(x => x.getClass.toString)
174+
assert(types.length == 9)
175+
assert(types(0).equals("class java.lang.String"))
176+
assert(types(1).equals("class java.lang.String"))
177+
assert(types(2).equals("class java.lang.String"))
178+
assert(types(3).equals("class java.lang.String"))
179+
assert(types(4).equals("class [B"))
180+
assert(types(5).equals("class [B"))
181+
assert(types(6).equals("class java.lang.String"))
182+
assert(types(7).equals("class java.lang.String"))
183+
assert(types(8).equals("class [B"))
184+
assert(row.getString(0).length == 10)
185+
assert(row.getString(0).trim.equals("the"))
186+
assert(row.getString(1).equals("quick"))
187+
assert(row.getString(2).length == 10)
188+
assert(row.getString(2).trim.equals("brown"))
189+
assert(row.getString(3).equals("fox"))
190+
assert(java.util.Arrays.equals(row.getAs[Array[Byte]](4), Array[Byte](0, 1, -30, 64)))
191+
assert(java.util.Arrays.equals(row.getAs[Array[Byte]](5), Array[Byte](0, 1, -30, 64)))
192+
assert(row.getString(6).equals("the"))
193+
assert(row.getString(7).equals("lazy"))
194+
assert(java.util.Arrays.equals(row.getAs[Array[Byte]](8), Array[Byte](100, 111, 103)))
195+
}
196+
197+
test("Basic write test") {
198+
val df1 = spark.read.jdbc(jdbcUrl, "numbers", new Properties)
199+
val df2 = spark.read.jdbc(jdbcUrl, "dates", new Properties)
200+
val df3 = spark.read.jdbc(jdbcUrl, "strings", new Properties)
201+
df1.write.jdbc(jdbcUrl, "numberscopy", new Properties)
202+
df2.write.jdbc(jdbcUrl, "datescopy", new Properties)
203+
df3.write.jdbc(jdbcUrl, "stringscopy", new Properties)
204+
}
205+
}

0 commit comments

Comments
 (0)