[SPARK-54057][BUILD] Retrieve dependency version from pom.xml for DependencyOverrides in SparkBuild.scala

### What changes were proposed in this pull request?
This PR proposes to retrieve dependency version from `pom.xml` for `DependencyOverrides` in `SparkBuild.scala`.

### Why are the changes needed?
Currently, version strings for some dependencies are hard coded in `DependencyOverrides` in `SparkBuild`, so developers need to keep consistent with version strings specified in `pom.xml` manually.

### Does this PR introduce _any_ user-facing change?
No.

### How was this patch tested?
GA.

### Was this patch authored or co-authored using generative AI tooling?
No.

Closes #52760 from sarutak/version-from-pom.

Authored-by: Kousuke Saruta <sarutak@amazon.co.jp>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>

Author: sarutak

pom.xml

@@ -183,6 +183,7 @@
     <scalafmt.skip>true</scalafmt.skip>
     <scalafmt.validateOnly>true</scalafmt.validateOnly>
     <scalafmt.changedOnly>true</scalafmt.changedOnly>
+    <!-- Should be consistent with SparkBuild.scala and docs -->
     <fasterxml.jackson.version>2.20.0</fasterxml.jackson.version>
     <ws.xmlschema.version>2.3.1</ws.xmlschema.version>
     <snappy.version>1.1.10.8</snappy.version>
@@ -219,6 +220,7 @@
     <netty-tcnative.version>2.0.74.Final</netty-tcnative.version>
     <icu4j.version>77.1</icu4j.version>
     <junit.version>6.0.0</junit.version>
+    <jline.version>2.14.6</jline.version>
     <!--
       SPARK-50299: When updating `sbt-jupiter-interface.version`,
       also need to update the version in `SparkBuild.scala` and `plugins.sbt`.
@@ -1180,7 +1182,7 @@
       <dependency>
         <groupId>jline</groupId>
         <artifactId>jline</artifactId>
-        <version>2.14.6</version>
+        <version>${jline.version}</version>
       </dependency>
       <dependency>
         <groupId>org.scalatest</groupId>

project/SparkBuild.scala

@@ -1186,15 +1186,26 @@ object KubernetesIntegrationTests {
  * Overrides to work around sbt's dependency resolution being different from Maven's.
  */
 object DependencyOverrides {
-  lazy val guavaVersion = sys.props.get("guava.version").getOrElse("33.4.0-jre")
   lazy val jacksonVersion = sys.props.get("fasterxml.jackson.version").getOrElse("2.20.0")
   lazy val jacksonDeps = Bom.dependencies("com.fasterxml.jackson" % "jackson-bom" % jacksonVersion)
   lazy val settings = jacksonDeps ++ Seq(
-    dependencyOverrides += "com.google.guava" % "guava" % guavaVersion,
-    dependencyOverrides ++= jacksonDeps.key.value,
-    dependencyOverrides += "jline" % "jline" % "2.14.6",
-    dependencyOverrides += "org.apache.avro" % "avro" % "1.12.1",
-    dependencyOverrides += "org.slf4j" % "slf4j-api" % "2.0.17")
+    dependencyOverrides ++= {
+      val guavaVersion = sys.props.get("guava.version").getOrElse(
+        SbtPomKeys.effectivePom.value.getProperties.get("guava.version").asInstanceOf[String])
+      val jlineVersion =
+        SbtPomKeys.effectivePom.value.getProperties.get("jline.version").asInstanceOf[String]
+      val avroVersion =
+        SbtPomKeys.effectivePom.value.getProperties.get("avro.version").asInstanceOf[String]
+      val slf4jVersion =
+        SbtPomKeys.effectivePom.value.getProperties.get("slf4j.version").asInstanceOf[String]
+      Seq(
+        "com.google.guava" % "guava" % guavaVersion,
+        "jline" % "jline" % jlineVersion,
+        "org.apache.avro" % "avro" % avroVersion,
+        "org.slf4j" % "slf4j-api" % slf4jVersion
+      ) ++ jacksonDeps.key.value
+    }
+  )
 }
 
 /**