Skip to content

Commit

Permalink
support spark 3.5.2/3
Browse files Browse the repository at this point in the history
  • Loading branch information
pang-wu committed Dec 8, 2024
1 parent 3fae143 commit dbc6e1f
Show file tree
Hide file tree
Showing 6 changed files with 17 additions and 367 deletions.
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,6 @@ examples/.ipynb_checkpoints/
*.parquet
*.crc
_SUCCESS

.metals/
.bloop/
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,10 @@ import com.intel.raydp.shims.{SparkShims, SparkShimDescriptor}
object SparkShimProvider {
val SPARK350_DESCRIPTOR = SparkShimDescriptor(3, 5, 0)
val SPARK351_DESCRIPTOR = SparkShimDescriptor(3, 5, 1)
val DESCRIPTOR_STRINGS = Seq(s"$SPARK350_DESCRIPTOR", s"$SPARK351_DESCRIPTOR")
val SPARK352_DESCRIPTOR = SparkShimDescriptor(3, 5, 2)
val SPARK353_DESCRIPTOR = SparkShimDescriptor(3, 5, 3)
val DESCRIPTOR_STRINGS = Seq(s"$SPARK350_DESCRIPTOR", s"$SPARK351_DESCRIPTOR", s"$SPARK352_DESCRIPTOR",
s"$SPARK353_DESCRIPTOR")
val DESCRIPTOR = SPARK350_DESCRIPTOR
}

Expand Down
7 changes: 0 additions & 7 deletions python/raydp/spark/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,3 @@
"from_spark_recoverable"
]

try:
import ray.util.data
from .dataset import RayMLDataset
__all__.append("RayMLDataset")
except ImportError:
# Ray MLDataset is removed in Ray 2.0
pass
Loading

0 comments on commit dbc6e1f

Please sign in to comment.