Address comments

huggingface · Apr 19, 2023 · 03f9d73 · 03f9d73
1 parent 0ba42c3
commit 03f9d73
Show file tree

Hide file tree

Showing 2 changed files with 1 addition and 4 deletions.
diff --git a/setup.py b/setup.py
@@ -137,8 +137,6 @@
     # Utilities from PyPA to e.g., compare versions
     "packaging",
     "responses<0.19",
-    # For from_spark
-    "pyspark",
     # To parse YAML metadata from dataset cards
     "pyyaml>=5.1",
 ]
@@ -171,6 +169,7 @@
     "elasticsearch<8.0.0",  # 8.0 asks users to provide hosts or cloud_id when instantiating ElasticSearch()
     "faiss-cpu>=1.6.4",
     "lz4",
+    "pyspark",
     "py7zr",
     "rarfile>=4.0",
     "sqlalchemy<2.0.0",

diff --git a/src/datasets/packaged_modules/__init__.py b/src/datasets/packaged_modules/__init__.py
@@ -9,7 +9,6 @@
 from .json import json
 from .pandas import pandas
 from .parquet import parquet
-from .spark import spark
 from .sql import sql  # noqa F401
 from .text import text
 
@@ -33,7 +32,6 @@ def _hash_python_lines(lines: List[str]) -> str:
     "json": (json.__name__, _hash_python_lines(inspect.getsource(json).splitlines())),
     "pandas": (pandas.__name__, _hash_python_lines(inspect.getsource(pandas).splitlines())),
     "parquet": (parquet.__name__, _hash_python_lines(inspect.getsource(parquet).splitlines())),
-    "spark": (spark.__name__, _hash_python_lines(inspect.getsource(spark).splitlines())),
     "text": (text.__name__, _hash_python_lines(inspect.getsource(text).splitlines())),
     "imagefolder": (imagefolder.__name__, _hash_python_lines(inspect.getsource(imagefolder).splitlines())),
     "audiofolder": (audiofolder.__name__, _hash_python_lines(inspect.getsource(audiofolder).splitlines())),