fixed download datasets for tests

sb-ai-lab · Aug 2, 2023 · 16cdab4 · 16cdab4
1 parent 76ab69a
commit 16cdab4
Show file tree

Hide file tree

Showing 9 changed files with 32 additions and 4,028 deletions.
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -36,6 +36,9 @@ jobs:
  pip3 install tox==3.24.4
  pip3 install tox-gh-actions==2.8.1
 
+ - name: download datasets
+ run: ./bin/download-datasets.sh
+
  - name: test with tox
  run: |
  tox
diff --git a/bin/download-datasets.sh b/bin/download-datasets.sh
@@ -2,7 +2,7 @@
 
 set -ex
 
-dataset_dir="/opt/spark_data/"
+dataset_dir="examples/data"
 
 mkdir -p "${dataset_dir}"
 
@@ -17,5 +17,4 @@ wget https://www.openml.org/data/get_csv/52422/ipums_la_97-small.arff -O ${datas
 
 head -n 25001 ${dataset_dir}/Buzzinsocialmedia_Twitter.csv > ${dataset_dir}/Buzzinsocialmedia_Twitter_25k.csv
 
-cp examples/data/sampled_app_train.csv ${dataset_dir}
-unzip examples/data/small_used_cars_data.zip -d ${dataset_dir}
+unzip ${dataset_dir}/small_used_cars_data.zip -d ${dataset_dir}
diff --git a/examples/data/.gitignore b/examples/data/.gitignore
@@ -1 +1,3 @@
 small_used_cars_data.csv
+*.csv
+!sampled_app_train.csv
diff --git a/examples/data/avito1k_train.csv b/examples/data/avito1k_train.csv
diff --git a/examples/data/tiny_used_cars_data.csv b/examples/data/tiny_used_cars_data.csv