From 7d75ee32b409f105df3800361c4a9a809372c8fb Mon Sep 17 00:00:00 2001 From: Luca Canali Date: Sun, 8 Oct 2023 09:56:51 -0700 Subject: [PATCH] [SPARK-45336][TEST] Use Oracle Database 23 Free for integration and testing ### What changes were proposed in this pull request? This proposes to update the Docker image used for integration tests and builds to Oracle Database 23c Free. ### Why are the changes needed? The Docker image used for integration tests and builds currently uses Oracle XE version 21.3.0. Oracle 21 support ends in April 2024. The latest Oracle release is 23c, it is a long-term release supported till 2032. With Oracle 23c, Oracle has changed the name of the free version of its database, from Oracle XE (Express Edition) to Oracle Database Free. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Existing test infrastructure. ### Was this patch authored or co-authored using generative AI tooling? No Closes #43123 from LucaCanali/useOracleFreeImage. Authored-by: Luca Canali Signed-off-by: Dongjoon Hyun --- .github/workflows/build_and_test.yml | 2 +- .../sql/jdbc/OracleIntegrationSuite.scala | 22 ++++++++++--------- .../sql/jdbc/v2/OracleIntegrationSuite.scala | 22 ++++++++++--------- .../sql/jdbc/v2/OracleNamespaceSuite.scala | 22 ++++++++++--------- 4 files changed, 37 insertions(+), 31 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 674e59508510c..401c58b2252b0 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -915,7 +915,7 @@ jobs: HIVE_PROFILE: hive2.3 GITHUB_PREV_SHA: ${{ github.event.before }} SPARK_LOCAL_IP: localhost - ORACLE_DOCKER_IMAGE_NAME: gvenzl/oracle-xe:21.3.0 + ORACLE_DOCKER_IMAGE_NAME: gvenzl/oracle-free:23.3 SKIP_MIMA: true SKIP_PACKAGING: true steps: diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala index 483f6087c81d2..4378c698883bf 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala @@ -41,23 +41,25 @@ import org.apache.spark.tags.DockerTest * - The documentation on how to build Oracle RDBMS in a container is at * https://github.com/oracle/docker-images/blob/master/OracleDatabase/SingleInstance/README.md * - Official Oracle container images can be found at https://container-registry.oracle.com - * - A trustable and streamlined Oracle XE database image can be found on Docker Hub at - * https://hub.docker.com/r/gvenzl/oracle-xe see also https://github.com/gvenzl/oci-oracle-xe + * - Trustable and streamlined Oracle Database Free images can be found on Docker Hub at + * https://hub.docker.com/r/gvenzl/oracle-free + * see also https://github.com/gvenzl/oci-oracle-free * 2. Run: export ORACLE_DOCKER_IMAGE_NAME=image_you_want_to_use_for_testing - * - Example: export ORACLE_DOCKER_IMAGE_NAME=gvenzl/oracle-xe:latest + * - Example: export ORACLE_DOCKER_IMAGE_NAME=gvenzl/oracle-free:latest * 3. Run: export ENABLE_DOCKER_INTEGRATION_TESTS=1 * 4. Start docker: sudo service docker start * - Optionally, docker pull $ORACLE_DOCKER_IMAGE_NAME * 5. Run Spark integration tests for Oracle with: ./build/sbt -Pdocker-integration-tests * "docker-integration-tests/testOnly org.apache.spark.sql.jdbc.OracleIntegrationSuite" * - * A sequence of commands to build the Oracle XE database container image: + * A sequence of commands to build the Oracle Database Free container image: * $ git clone https://github.com/oracle/docker-images.git * $ cd docker-images/OracleDatabase/SingleInstance/dockerfiles - * $ ./buildContainerImage.sh -v 21.3.0 -x - * $ export ORACLE_DOCKER_IMAGE_NAME=oracle/database:21.3.0-xe + * $ ./buildContainerImage.sh -v 23.2.0 -f + * $ export ORACLE_DOCKER_IMAGE_NAME=oracle/database:23.2.0-free * - * This procedure has been validated with Oracle 18.4.0 and 21.3.0 Express Edition. + * This procedure has been validated with Oracle Databae Free version 23.2.0, + * and with Oracle Express Edition versions 18.4.0 and 21.3.0 */ @DockerTest class OracleIntegrationSuite extends DockerJDBCIntegrationSuite with SharedSparkSession { @@ -65,16 +67,16 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationSuite with SharedSpark override val db = new DatabaseOnDocker { lazy override val imageName = - sys.env.getOrElse("ORACLE_DOCKER_IMAGE_NAME", "gvenzl/oracle-xe:21.3.0") + sys.env.getOrElse("ORACLE_DOCKER_IMAGE_NAME", "gvenzl/oracle-free:23.3") val oracle_password = "Th1s1sThe0racle#Pass" override val env = Map( "ORACLE_PWD" -> oracle_password, // oracle images uses this - "ORACLE_PASSWORD" -> oracle_password // gvenzl/oracle-xe uses this + "ORACLE_PASSWORD" -> oracle_password // gvenzl/oracle-free uses this ) override val usesIpc = false override val jdbcPort: Int = 1521 override def getJdbcUrl(ip: String, port: Int): String = - s"jdbc:oracle:thin:system/$oracle_password@//$ip:$port/xe" + s"jdbc:oracle:thin:system/$oracle_password@//$ip:$port/freepdb1" } override val connectionTimeout = timeout(7.minutes) diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/OracleIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/OracleIntegrationSuite.scala index 5124199328ce2..83ba1e1592ccb 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/OracleIntegrationSuite.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/OracleIntegrationSuite.scala @@ -36,23 +36,25 @@ import org.apache.spark.tags.DockerTest * - The documentation on how to build Oracle RDBMS in a container is at * https://github.com/oracle/docker-images/blob/master/OracleDatabase/SingleInstance/README.md * - Official Oracle container images can be found at https://container-registry.oracle.com - * - A trustable and streamlined Oracle XE database image can be found on Docker Hub at - * https://hub.docker.com/r/gvenzl/oracle-xe see also https://github.com/gvenzl/oci-oracle-xe + * - Trustable and streamlined Oracle Database Free images can be found on Docker Hub at + * https://hub.docker.com/r/gvenzl/oracle-free + * see also https://github.com/gvenzl/oci-oracle-free * 2. Run: export ORACLE_DOCKER_IMAGE_NAME=image_you_want_to_use_for_testing - * - Example: export ORACLE_DOCKER_IMAGE_NAME=gvenzl/oracle-xe:latest + * - Example: export ORACLE_DOCKER_IMAGE_NAME=gvenzl/oracle-free:latest * 3. Run: export ENABLE_DOCKER_INTEGRATION_TESTS=1 * 4. Start docker: sudo service docker start * - Optionally, docker pull $ORACLE_DOCKER_IMAGE_NAME * 5. Run Spark integration tests for Oracle with: ./build/sbt -Pdocker-integration-tests * "testOnly org.apache.spark.sql.jdbc.v2.OracleIntegrationSuite" * - * A sequence of commands to build the Oracle XE database container image: + * A sequence of commands to build the Oracle Database Free container image: * $ git clone https://github.com/oracle/docker-images.git * $ cd docker-images/OracleDatabase/SingleInstance/dockerfiles - * $ ./buildContainerImage.sh -v 21.3.0 -x - * $ export ORACLE_DOCKER_IMAGE_NAME=oracle/database:21.3.0-xe + * $ ./buildContainerImage.sh -v 23.2.0 -f + * $ export ORACLE_DOCKER_IMAGE_NAME=oracle/database:23.2.0-free * - * This procedure has been validated with Oracle 18.4.0 and 21.3.0 Express Edition. + * This procedure has been validated with Oracle Databae Free version 23.2.0, + * and with Oracle Express Edition versions 18.4.0 and 21.3.0 */ @DockerTest class OracleIntegrationSuite extends DockerJDBCIntegrationV2Suite with V2JDBCTest { @@ -74,16 +76,16 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationV2Suite with V2JDBCTes override val namespaceOpt: Option[String] = Some("SYSTEM") override val db = new DatabaseOnDocker { lazy override val imageName = - sys.env.getOrElse("ORACLE_DOCKER_IMAGE_NAME", "gvenzl/oracle-xe:21.3.0") + sys.env.getOrElse("ORACLE_DOCKER_IMAGE_NAME", "gvenzl/oracle-free:23.3") val oracle_password = "Th1s1sThe0racle#Pass" override val env = Map( "ORACLE_PWD" -> oracle_password, // oracle images uses this - "ORACLE_PASSWORD" -> oracle_password // gvenzl/oracle-xe uses this + "ORACLE_PASSWORD" -> oracle_password // gvenzl/oracle-free uses this ) override val usesIpc = false override val jdbcPort: Int = 1521 override def getJdbcUrl(ip: String, port: Int): String = - s"jdbc:oracle:thin:system/$oracle_password@//$ip:$port/xe" + s"jdbc:oracle:thin:system/$oracle_password@//$ip:$port/freepdb1" } override def sparkConf: SparkConf = super.sparkConf diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/OracleNamespaceSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/OracleNamespaceSuite.scala index a365a1c4e82e4..c1e5023de5e4f 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/OracleNamespaceSuite.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/OracleNamespaceSuite.scala @@ -32,23 +32,25 @@ import org.apache.spark.tags.DockerTest * - The documentation on how to build Oracle RDBMS in a container is at * https://github.com/oracle/docker-images/blob/master/OracleDatabase/SingleInstance/README.md * - Official Oracle container images can be found at https://container-registry.oracle.com - * - A trustable and streamlined Oracle XE database image can be found on Docker Hub at - * https://hub.docker.com/r/gvenzl/oracle-xe see also https://github.com/gvenzl/oci-oracle-xe + * - Trustable and streamlined Oracle Database Free images can be found on Docker Hub at + * https://hub.docker.com/r/gvenzl/oracle-free + * see also https://github.com/gvenzl/oci-oracle-free * 2. Run: export ORACLE_DOCKER_IMAGE_NAME=image_you_want_to_use_for_testing - * - Example: export ORACLE_DOCKER_IMAGE_NAME=gvenzl/oracle-xe:latest + * - Example: export ORACLE_DOCKER_IMAGE_NAME=gvenzl/oracle-free:latest * 3. Run: export ENABLE_DOCKER_INTEGRATION_TESTS=1 * 4. Start docker: sudo service docker start * - Optionally, docker pull $ORACLE_DOCKER_IMAGE_NAME * 5. Run Spark integration tests for Oracle with: ./build/sbt -Pdocker-integration-tests * "testOnly org.apache.spark.sql.jdbc.v2.OracleNamespaceSuite" * - * A sequence of commands to build the Oracle XE database container image: + * A sequence of commands to build the Oracle Database Free container image: * $ git clone https://github.com/oracle/docker-images.git * $ cd docker-images/OracleDatabase/SingleInstance/dockerfiles - * $ ./buildContainerImage.sh -v 21.3.0 -x - * $ export ORACLE_DOCKER_IMAGE_NAME=oracle/database:21.3.0-xe + * $ ./buildContainerImage.sh -v 23.2.0 -f + * $ export ORACLE_DOCKER_IMAGE_NAME=oracle/database:23.2.0-free * - * This procedure has been validated with Oracle 18.4.0 and 21.3.0 Express Edition. + * This procedure has been validated with Oracle Databae Free version 23.2.0, + * and with Oracle Express Edition versions 18.4.0 and 21.3.0 */ @DockerTest class OracleNamespaceSuite extends DockerJDBCIntegrationSuite with V2JDBCNamespaceTest { @@ -57,16 +59,16 @@ class OracleNamespaceSuite extends DockerJDBCIntegrationSuite with V2JDBCNamespa override val db = new DatabaseOnDocker { lazy override val imageName = - sys.env.getOrElse("ORACLE_DOCKER_IMAGE_NAME", "gvenzl/oracle-xe:21.3.0") + sys.env.getOrElse("ORACLE_DOCKER_IMAGE_NAME", "gvenzl/oracle-free:23.3") val oracle_password = "Th1s1sThe0racle#Pass" override val env = Map( "ORACLE_PWD" -> oracle_password, // oracle images uses this - "ORACLE_PASSWORD" -> oracle_password // gvenzl/oracle-xe uses this + "ORACLE_PASSWORD" -> oracle_password // gvenzl/oracle-free uses this ) override val usesIpc = false override val jdbcPort: Int = 1521 override def getJdbcUrl(ip: String, port: Int): String = - s"jdbc:oracle:thin:system/$oracle_password@//$ip:$port/xe" + s"jdbc:oracle:thin:system/$oracle_password@//$ip:$port/freepdb1" } val map = new CaseInsensitiveStringMap(