From 1c6d64e6b23897f799417abff547a19f70e3c493 Mon Sep 17 00:00:00 2001 From: chengmengli06 Date: Thu, 17 Nov 2022 17:35:05 +0800 Subject: [PATCH] fix negative sample using hive input bug, mis classify local .csv inputs as hive inputs --- easy_rec/python/utils/config_util.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/easy_rec/python/utils/config_util.py b/easy_rec/python/utils/config_util.py index 2ab5c751d..f64455b14 100644 --- a/easy_rec/python/utils/config_util.py +++ b/easy_rec/python/utils/config_util.py @@ -537,9 +537,12 @@ def process_data_path(data_path, hive_util): def process_neg_sampler_data_path(pipeline_config): # replace neg_sampler hive table => hdfs path if pai_util.is_on_pai(): - return None + return if not pipeline_config.data_config.HasField('sampler'): - return None + return + # not using hive, so not need to process it + if pipeline_config.WhichOneof('train_path') != 'hive_train_input': + return hive_util = HiveUtils( data_config=pipeline_config.data_config, hive_config=pipeline_config.hive_train_input)