From 03f9d737190cda4252935865b3e441b55e9c3522 Mon Sep 17 00:00:00 2001 From: Maddie Dawson Date: Wed, 19 Apr 2023 11:41:10 -0700 Subject: [PATCH] Address comments --- setup.py | 3 +-- src/datasets/packaged_modules/__init__.py | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 20ce01ebffbb..b0ad90ca5509 100644 --- a/setup.py +++ b/setup.py @@ -137,8 +137,6 @@ # Utilities from PyPA to e.g., compare versions "packaging", "responses<0.19", - # For from_spark - "pyspark", # To parse YAML metadata from dataset cards "pyyaml>=5.1", ] @@ -171,6 +169,7 @@ "elasticsearch<8.0.0", # 8.0 asks users to provide hosts or cloud_id when instantiating ElasticSearch() "faiss-cpu>=1.6.4", "lz4", + "pyspark", "py7zr", "rarfile>=4.0", "sqlalchemy<2.0.0", diff --git a/src/datasets/packaged_modules/__init__.py b/src/datasets/packaged_modules/__init__.py index f5a48ff868d0..f3553b0b9618 100644 --- a/src/datasets/packaged_modules/__init__.py +++ b/src/datasets/packaged_modules/__init__.py @@ -9,7 +9,6 @@ from .json import json from .pandas import pandas from .parquet import parquet -from .spark import spark from .sql import sql # noqa F401 from .text import text @@ -33,7 +32,6 @@ def _hash_python_lines(lines: List[str]) -> str: "json": (json.__name__, _hash_python_lines(inspect.getsource(json).splitlines())), "pandas": (pandas.__name__, _hash_python_lines(inspect.getsource(pandas).splitlines())), "parquet": (parquet.__name__, _hash_python_lines(inspect.getsource(parquet).splitlines())), - "spark": (spark.__name__, _hash_python_lines(inspect.getsource(spark).splitlines())), "text": (text.__name__, _hash_python_lines(inspect.getsource(text).splitlines())), "imagefolder": (imagefolder.__name__, _hash_python_lines(inspect.getsource(imagefolder).splitlines())), "audiofolder": (audiofolder.__name__, _hash_python_lines(inspect.getsource(audiofolder).splitlines())),