[feat] Make hdfs function compatible with tensorflow-io and TensorFlo…

…w version earlier than 2.6 by using FileSystem in TensorFlow Env.

Change hdfs file system to all file system.
Now also support S3, local and so on.

Also change Redis backend function to make it more low coupling with TF.

Also modify GPU/CPU hash table code(rehashifneeded/dump/cpu insert) to make them more decoupled.

.github/workflows/make_wheel_Windows_x86.sh

@@ -8,10 +8,11 @@ if [ -z $HOROVOD_VERSION ] ; then
 fi
 
 python -m pip install --default-timeout=1000 wheel setuptools tensorflow==$TF_VERSION horovod==$HOROVOD_VERSION
-bash ./tools/testing/build_and_run_tests.sh
-
+python -m pip install tensorflow-io
 python -m pip install --upgrade protobuf==3.20.0
 
+bash ./tools/testing/build_and_run_tests.sh
+
 python configure.py
 
 bazel.exe build  --no-cache \

.github/workflows/make_wheel_macOS_arm64.sh

@@ -11,7 +11,8 @@ export TF_NEED_CUDA=0
 
 python --version
 
-RUN python -m pip install --upgrade protobuf==3.20.0
+python -m pip install tensorflow-io
+python -m pip install --upgrade protobuf==3.20.0
 python configure.py
 
 bazel build \

.github/workflows/make_wheel_macOS_x86.sh

@@ -9,6 +9,7 @@ python --version
 brew install open-mpi
 
 python -m pip install --default-timeout=1000 delocate==0.9.1 wheel setuptools tensorflow==$TF_VERSION
+python -m pip install tensorflow-io
 python -m pip install --upgrade protobuf==3.20.0
 
 bash tools/docker/install/install_horovod.sh $HOROVOD_VERSION --only-cpu

.github/workflows/release.yml

@@ -34,6 +34,7 @@ jobs:
           pip install --default-timeout=1000 -r tools/install_deps/pytest.txt -r tools/install_deps/tensorflow-cpu.txt -r requirements.txt
           sudo apt install -y redis > /dev/null 2> /dev/null
           bash tools/install_deps/install_bazelisk.sh ./
+          python -m pip install tensorflow-io
           python -m pip install --upgrade protobuf==3.20.0
           python configure.py
           bazel test --local_ram_resources=4096 -c opt -k --test_timeout 300,450,1200,3600 --test_output=errors //tensorflow_recommenders_addons/...

WORKSPACE

@@ -54,16 +54,6 @@ http_archive(
     url = "https://github.com/sewenew/redis-plus-plus/archive/refs/tags/1.2.3.zip",
 )
 
-http_archive(
-    name = "hadoop",
-    build_file = "//third_party:hadoop.BUILD",
-    sha256 = "fa9d0587d06c36838e778081bcf8271a9c63060af00b3bf456423c1777a62043",
-    strip_prefix = "hadoop-rel-release-3.3.0",
-    urls = [
-        "https://github.com/apache/hadoop/archive/refs/tags/rel/release-3.3.0.tar.gz",
-    ],
-)
-
 tf_configure(
     name = "local_config_tf",
 )

tensorflow_recommenders_addons/dynamic_embedding/core/BUILD

@@ -31,7 +31,6 @@ custom_op_library(
     ] + glob(["kernels/lookup_impl/lookup_table_op_gpu*"])),
     deps = [
         "//tensorflow_recommenders_addons/dynamic_embedding/core/lib/cuckoo:cuckoohash",
-        "//tensorflow_recommenders_addons/dynamic_embedding/core/lib/hadoop_file_system",
     ],
 )