[SPARK-54076][SQL][TESTS] Use sarutak/oracle-free image to use PASSWORD_INIT_TIMEOUT for OracleDatabaseOnDocker

sarutak · dongjoon-hyun · commit 576f9a515976 · 2025-10-29T10:17:06.000-07:00
### What changes were proposed in this pull request? This PR proposes to replace `gvenzl`'s Oracle docker image for the integration test with [my customized one](https://github.com/sarutak/oci-oracle-free/tree/password-initialization-timeout) to fix the flaky test `OracleJoinPushdownIntegrationSuite`. Recently, this test frequently fails. https://github.com/beliefer/spark/actions/runs/18904457292/job/53959322233#logs ``` [info] org.apache.spark.sql.jdbc.v2.join.OracleJoinPushdownIntegrationSuite *** ABORTED *** (10 minutes, 15 seconds) [info] The code passed to eventually never returned normally. Attempted 599 times over 10.012648813883333 minutes. Last failure message: ORA-12541: Cannot connect. No listener at host 10.1.0.201 port 44551. (CONNECTION_ID=5rB4vZNTQR2sFHpk9xQC1A==) [info] https://docs.oracle.com/error-help/db/ora-12541/. (DockerJDBCIntegrationSuite.scala:214) [info] org.scalatest.exceptions.TestFailedDueToTimeoutException: [info] at org.scalatest.enablers.Retrying$$anon$4.tryTryAgain$2(Retrying.scala:219) [info] at org.scalatest.enablers.Retrying$$anon$4.retry(Retrying.scala:226) [info] at org.scalatest.concurrent.Eventually.eventually(Eventually.scala:313) [info] at org.scalatest.concurrent.Eventually.eventually$(Eventually.scala:312) [info] at org.apache.spark.sql.jdbc.DockerJDBCIntegrationSuite.eventually(DockerJDBCIntegrationSuite.scala:103) [info] at org.apache.spark.sql.jdbc.DockerJDBCIntegrationSuite.$anonfun$beforeAll$1(DockerJDBCIntegrationSuite.scala:214) [info] at org.apache.spark.sql.jdbc.DockerIntegrationFunSuite.runIfTestsEnabled(DockerIntegrationFunSuite.scala:49) [info] at org.apache.spark.sql.jdbc.DockerIntegrationFunSuite.runIfTestsEnabled$(DockerIntegrationFunSuite.scala:47) [info] at org.apache.spark.sql.jdbc.DockerJDBCIntegrationSuite.runIfTestsEnabled(DockerJDBCIntegrationSuite.scala:103) [info] at org.apache.spark.sql.jdbc.DockerJDBCIntegrationSuite.beforeAll(DockerJDBCIntegrationSuite.scala:133) [info] at org.scalatest.BeforeAndAfterAll.liftedTree1$1(BeforeAndAfterAll.scala:212) [info] at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:210) [info] at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208) [info] at org.apache.spark.SparkFunSuite.run(SparkFunSuite.scala:68) [info] at org.scalatest.tools.Framework.org$scalatest$tools$Framework$$runSuite(Framework.scala:321) [info] at org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:517) [info] at sbt.ForkMain$Run.lambda$runTest$1(ForkMain.java:414) [info] at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) [info] at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [info] at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [info] at java.base/java.lang.Thread.run(Thread.java:840) [info] Cause: java.sql.SQLException: ORA-12541: Cannot connect. No listener at host 10.1.0.201 port 44551. (CONNECTION_ID=5rB4vZNTQR2sFHpk9xQC1A==) [info] https://docs.oracle.com/error-help/db/ora-12541/ [info] at oracle.jdbc.driver.T4CConnection.handleLogonNetException(T4CConnection.java:1631) [info] at oracle.jdbc.driver.T4CConnection.logon(T4CConnection.java:1151) [info] at oracle.jdbc.driver.PhysicalConnection.connect(PhysicalConnection.java:1189) [info] at oracle.jdbc.driver.T4CDriverExtension.getConnection(T4CDriverExtension.java:106) [info] at oracle.jdbc.driver.OracleDriver.connect(OracleDriver.java:895) [info] at oracle.jdbc.driver.OracleDriver.connect(OracleDriver.java:702) [info] at java.sql/java.sql.DriverManager.getConnection(DriverManager.java:681) [info] at java.sql/java.sql.DriverManager.getConnection(DriverManager.java:190) [info] at org.apache.spark.sql.jdbc.DockerJDBCIntegrationSuite.getConnection(DockerJDBCIntegrationSuite.scala:250) [info] at org.apache.spark.sql.jdbc.DockerJDBCIntegrationSuite.$anonfun$beforeAll$8(DockerJDBCIntegrationSuite.scala:215) [info] at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) [info] at org.scalatest.enablers.Retrying$$anon$4.makeAValiantAttempt$1(Retrying.scala:184) [info] at org.scalatest.enablers.Retrying$$anon$4.tryTryAgain$2(Retrying.scala:196) [info] at org.scalatest.enablers.Retrying$$anon$4.retry(Retrying.scala:226) [info] at org.scalatest.concurrent.Eventually.eventually(Eventually.scala:313) [info] at org.scalatest.concurrent.Eventually.eventually$(Eventually.scala:312) [info] at org.apache.spark.sql.jdbc.DockerJDBCIntegrationSuite.eventually(DockerJDBCIntegrationSuite.scala:103) [info] at org.apache.spark.sql.jdbc.DockerJDBCIntegrationSuite.$anonfun$beforeAll$1(DockerJDBCIntegrationSuite.scala:214) [info] at org.apache.spark.sql.jdbc.DockerIntegrationFunSuite.runIfTestsEnabled(DockerIntegrationFunSuite.scala:49) [info] at org.apache.spark.sql.jdbc.DockerIntegrationFunSuite.runIfTestsEnabled$(DockerIntegrationFunSuite.scala:47) [info] at org.apache.spark.sql.jdbc.DockerJDBCIntegrationSuite.runIfTestsEnabled(DockerJDBCIntegrationSuite.scala:103) [info] at org.apache.spark.sql.jdbc.DockerJDBCIntegrationSuite.beforeAll(DockerJDBCIntegrationSuite.scala:133) [info] at org.scalatest.BeforeAndAfterAll.liftedTree1$1(BeforeAndAfterAll.scala:212) [info] at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:210) [info] at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208) [info] at org.apache.spark.SparkFunSuite.run(SparkFunSuite.scala:68) [info] at org.scalatest.tools.Framework.org$scalatest$tools$Framework$$runSuite(Framework.scala:321) [info] at org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:517) [info] at sbt.ForkMain$Run.lambda$runTest$1(ForkMain.java:414) [info] at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) [info] at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [info] at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [info] at java.base/java.lang.Thread.run(Thread.java:840) [info] Cause: oracle.net.ns.NetException: ORA-12541: Cannot connect. No listener at host 10.1.0.201 port 44551. (CONNECTION_ID=5rB4vZNTQR2sFHpk9xQC1A==) [info] https://docs.oracle.com/error-help/db/ora-12541/ [info] at oracle.net.nt.TcpNTAdapter.handleEstablishSocketException(TcpNTAdapter.java:418) [info] at oracle.net.nt.TcpNTAdapter.establishSocket(TcpNTAdapter.java:350) [info] at oracle.net.nt.TcpNTAdapter.connect(TcpNTAdapter.java:228) [info] at oracle.net.nt.ConnOption.connect(ConnOption.java:333) [info] at oracle.net.nt.ConnStrategy.executeConnOption(ConnStrategy.java:1223) [info] at oracle.net.nt.ConnStrategy.execute(ConnStrategy.java:762) [info] at oracle.net.resolver.AddrResolution.resolveAndExecute(AddrResolution.java:712) [info] at oracle.net.ns.NSProtocol.establishConnection(NSProtocol.java:960) [info] at oracle.net.ns.NSProtocol.connect(NSProtocol.java:329) [info] at oracle.jdbc.driver.T4CConnection.connectNetworkSessionProtocol(T4CConnection.java:3462) [info] at oracle.jdbc.driver.T4CConnection.logon(T4CConnection.java:1030) [info] at oracle.jdbc.driver.PhysicalConnection.connect(PhysicalConnection.java:1189) [info] at oracle.jdbc.driver.T4CDriverExtension.getConnection(T4CDriverExtension.java:106) [info] at oracle.jdbc.driver.OracleDriver.connect(OracleDriver.java:895) [info] at oracle.jdbc.driver.OracleDriver.connect(OracleDriver.java:702) [info] at java.sql/java.sql.DriverManager.getConnection(DriverManager.java:681) [info] at java.sql/java.sql.DriverManager.getConnection(DriverManager.java:190) [info] at org.apache.spark.sql.jdbc.DockerJDBCIntegrationSuite.getConnection(DockerJDBCIntegrationSuite.scala:250) [info] at org.apache.spark.sql.jdbc.DockerJDBCIntegrationSuite.$anonfun$beforeAll$8(DockerJDBCIntegrationSuite.scala:215) [info] at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) [info] at org.scalatest.enablers.Retrying$$anon$4.makeAValiantAttempt$1(Retrying.scala:184) [info] at org.scalatest.enablers.Retrying$$anon$4.tryTryAgain$2(Retrying.scala:196) [info] at org.scalatest.enablers.Retrying$$anon$4.retry(Retrying.scala:226) [info] at org.scalatest.concurrent.Eventually.eventually(Eventually.scala:313) [info] at org.scalatest.concurrent.Eventually.eventually$(Eventually.scala:312) [info] at org.apache.spark.sql.jdbc.DockerJDBCIntegrationSuite.eventually(DockerJDBCIntegrationSuite.scala:103) [info] at org.apache.spark.sql.jdbc.DockerJDBCIntegrationSuite.$anonfun$beforeAll$1(DockerJDBCIntegrationSuite.scala:214) [info] at org.apache.spark.sql.jdbc.DockerIntegrationFunSuite.runIfTestsEnabled(DockerIntegrationFunSuite.scala:49) [info] at org.apache.spark.sql.jdbc.DockerIntegrationFunSuite.runIfTestsEnabled$(DockerIntegrationFunSuite.scala:47) [info] at org.apache.spark.sql.jdbc.DockerJDBCIntegrationSuite.runIfTestsEnabled(DockerJDBCIntegrationSuite.scala:103) [info] at org.apache.spark.sql.jdbc.DockerJDBCIntegrationSuite.beforeAll(DockerJDBCIntegrationSuite.scala:133) [info] at org.scalatest.BeforeAndAfterAll.liftedTree1$1(BeforeAndAfterAll.scala:212) [info] at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:210) [info] at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208) [info] at org.apache.spark.SparkFunSuite.run(SparkFunSuite.scala:68) [info] at org.scalatest.tools.Framework.org$scalatest$tools$Framework$$runSuite(Framework.scala:321) [info] at org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:517) [info] at sbt.ForkMain$Run.lambda$runTest$1(ForkMain.java:414) [info] at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) [info] at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [info] at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [info] at java.base/java.lang.Thread.run(Thread.java:840) [info] Cause: java.net.ConnectException: Connection refused [info] at java.base/sun.nio.ch.Net.pollConnect(Native Method) [info] at java.base/sun.nio.ch.Net.pollConnectNow(Net.java:672) [info] at java.base/sun.nio.ch.SocketChannelImpl.finishTimedConnect(SocketChannelImpl.java:1141) [info] at java.base/sun.nio.ch.SocketChannelImpl.blockingConnect(SocketChannelImpl.java:1183) [info] at java.base/sun.nio.ch.SocketAdaptor.connect(SocketAdaptor.java:98) [info] at oracle.net.nt.TimeoutSocketChannel.doConnect(TimeoutSocketChannel.java:289) [info] at oracle.net.nt.TimeoutSocketChannel.initializeSocketChannel(TimeoutSocketChannel.java:269) [info] at oracle.net.nt.TimeoutSocketChannel.connect(TimeoutSocketChannel.java:236) [info] at oracle.net.nt.TimeoutSocketChannel.<init>(TimeoutSocketChannel.java:203) [info] at oracle.net.nt.TcpNTAdapter.establishSocket(TcpNTAdapter.java:339) [info] at oracle.net.nt.TcpNTAdapter.connect(TcpNTAdapter.java:228) [info] at oracle.net.nt.ConnOption.connect(ConnOption.java:333) [info] at oracle.net.nt.ConnStrategy.executeConnOption(ConnStrategy.java:1223) [info] at oracle.net.nt.ConnStrategy.execute(ConnStrategy.java:762) [info] at oracle.net.resolver.AddrResolution.resolveAndExecute(AddrResolution.java:712) [info] at oracle.net.ns.NSProtocol.establishConnection(NSProtocol.java:960) [info] at oracle.net.ns.NSProtocol.connect(NSProtocol.java:329) [info] at oracle.jdbc.driver.T4CConnection.connectNetworkSessionProtocol(T4CConnection.java:3462) [info] at oracle.jdbc.driver.T4CConnection.logon(T4CConnection.java:1030) [info] at oracle.jdbc.driver.PhysicalConnection.connect(PhysicalConnection.java:1189) [info] at oracle.jdbc.driver.T4CDriverExtension.getConnection(T4CDriverExtension.java:106) [info] at oracle.jdbc.driver.OracleDriver.connect(OracleDriver.java:895) [info] at oracle.jdbc.driver.OracleDriver.connect(OracleDriver.java:702) [info] at java.sql/java.sql.DriverManager.getConnection(DriverManager.java:681) [info] at java.sql/java.sql.DriverManager.getConnection(DriverManager.java:190) [info] at org.apache.spark.sql.jdbc.DockerJDBCIntegrationSuite.getConnection(DockerJDBCIntegrationSuite.scala:250) [info] at org.apache.spark.sql.jdbc.DockerJDBCIntegrationSuite.$anonfun$beforeAll$8(DockerJDBCIntegrationSuite.scala:215) [info] at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) [info] at org.scalatest.enablers.Retrying$$anon$4.makeAValiantAttempt$1(Retrying.scala:184) [info] at org.scalatest.enablers.Retrying$$anon$4.tryTryAgain$2(Retrying.scala:196) [info] at org.scalatest.enablers.Retrying$$anon$4.retry(Retrying.scala:226) [info] at org.scalatest.concurrent.Eventually.eventually(Eventually.scala:313) [info] at org.scalatest.concurrent.Eventually.eventually$(Eventually.scala:312) [info] at org.apache.spark.sql.jdbc.DockerJDBCIntegrationSuite.eventually(DockerJDBCIntegrationSuite.scala:103) [info] at org.apache.spark.sql.jdbc.DockerJDBCIntegrationSuite.$anonfun$beforeAll$1(DockerJDBCIntegrationSuite.scala:214) [info] at org.apache.spark.sql.jdbc.DockerIntegrationFunSuite.runIfTestsEnabled(DockerIntegrationFunSuite.scala:49) [info] at org.apache.spark.sql.jdbc.DockerIntegrationFunSuite.runIfTestsEnabled$(DockerIntegrationFunSuite.scala:47) [info] at org.apache.spark.sql.jdbc.DockerJDBCIntegrationSuite.runIfTestsEnabled(DockerJDBCIntegrationSuite.scala:103) [info] at org.apache.spark.sql.jdbc.DockerJDBCIntegrationSuite.beforeAll(DockerJDBCIntegrationSuite.scala:133) [info] at org.scalatest.BeforeAndAfterAll.liftedTree1$1(BeforeAndAfterAll.scala:212) [info] at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:210) [info] at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208) [info] at org.apache.spark.SparkFunSuite.run(SparkFunSuite.scala:68) [info] at org.scalatest.tools.Framework.org$scalatest$tools$Framework$$runSuite(Framework.scala:321) [info] at org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:517) [info] at sbt.ForkMain$Run.lambda$runTest$1(ForkMain.java:414) [info] at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) [info] at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [info] at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [info] at java.base/java.lang.Thread.run(Thread.java:840) ``` I sometimes reproduce this issue on my local dev environment and the cause seems that after initializing password for `SYS`, DDL lock can still be held, leading password initialization for `SYSTEM` fails (the default value of `DDL_TIMEOUT` seems 0). ``` CONTAINER: starting up... CONTAINER: first database startup, initializing... CONTAINER: uncompressing database data files, please wait... CONTAINER: done uncompressing database data files, duration: 6 seconds. CONTAINER: starting up Oracle Database... LSNRCTL for Linux: Version 23.0.0.0.0 - Production on 07-OCT-2025 01:28:46 Copyright (c) 1991, 2025, Oracle. All rights reserved. Starting /opt/oracle/product/23ai/dbhomeFree/bin/tnslsnr: please wait... TNSLSNR for Linux: Version 23.0.0.0.0 - Production System parameter file is /opt/oracle/product/23ai/dbhomeFree/network/admin/listener.ora Log messages written to /opt/oracle/diag/tnslsnr/6f4b55693218/listener/alert/log.xml Listening on: (DESCRIPTION=(ADDRESS=(PROTOCOL=ipc)(KEY=EXTPROC_FOR_FREE))) Listening on: (DESCRIPTION=(ADDRESS=(PROTOCOL=tcp)(HOST=0.0.0.0)(PORT=1521))) Connecting to (DESCRIPTION=(ADDRESS=(PROTOCOL=IPC)(KEY=EXTPROC_FOR_FREE))) STATUS of the LISTENER ------------------------ Alias LISTENER Version TNSLSNR for Linux: Version 23.0.0.0.0 - Production Start Date 07-OCT-2025 01:28:46 Uptime 0 days 0 hr. 0 min. 0 sec Trace Level off Security ON: Local OS Authentication SNMP OFF Default Service FREE Listener Parameter File /opt/oracle/product/23ai/dbhomeFree/network/admin/listener.ora Listener Log File /opt/oracle/diag/tnslsnr/6f4b55693218/listener/alert/log.xml Listening Endpoints Summary... (DESCRIPTION=(ADDRESS=(PROTOCOL=ipc)(KEY=EXTPROC_FOR_FREE))) (DESCRIPTION=(ADDRESS=(PROTOCOL=tcp)(HOST=0.0.0.0)(PORT=1521))) The listener supports no services The command completed successfully ORACLE instance started. Total System Global Area 1603293992 bytes Fixed Size 4928296 bytes Variable Size 637534208 bytes Database Buffers 956301312 bytes Redo Buffers 4530176 bytes Database mounted. Database opened. CONTAINER: Resetting SYS and SYSTEM passwords. User altered. ALTER USER SYSTEM IDENTIFIED BY "Th1s1sThe0racle#Pass" * ERROR at line 1: ORA-65048: error encountered when processing the current DDL statement in pluggable database FREEPDB1 ORA-04021: timeout occurred while waiting to lock object Help: https://docs.oracle.com/error-help/db/ora-65048/ ``` The custom docker image allows to set timeout for password initialization through an environment variable `PASSWORD_INIT_TIMEOUT` and set the default value to `30` for the integration test. ### Why are the changes needed? For test stability. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? GA. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #52780 from sarutak/password-timeout-oracle-image. Authored-by: Kousuke Saruta <sarutak@apache.org> Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleDatabaseOnDocker.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleDatabaseOnDocker.scala
@@ -20,12 +20,15 @@ package org.apache.spark.sql.jdbc
 import org.apache.spark.internal.Logging
 
 class OracleDatabaseOnDocker extends DatabaseOnDocker with Logging {
+  // sarutak/oracle-free is a custom fork of gvenzl/oracle-free which allows to set timeout for
+  // password initialization. See SPARK-54076 for details.
   lazy override val imageName =
-    sys.env.getOrElse("ORACLE_DOCKER_IMAGE_NAME", "gvenzl/oracle-free:23.9-slim")
+    sys.env.getOrElse("ORACLE_DOCKER_IMAGE_NAME", "sarutak/oracle-free:23.9-slim")
   val oracle_password = "Th1s1sThe0racle#Pass"
   override val env = Map(
     "ORACLE_PWD" -> oracle_password, // oracle images uses this
-    "ORACLE_PASSWORD" -> oracle_password // gvenzl/oracle-free uses this
+    "ORACLE_PASSWORD" -> oracle_password, // gvenzl/oracle-free uses this
+    "PASSWORD_INIT_TIMEOUT" -> "30"
   )
   override val usesIpc = false
   override val jdbcPort: Int = 1521
