From 1c0db6451fb5412aca00892930ccff6e551ad145 Mon Sep 17 00:00:00 2001 From: Moritz Meister <8422705+moritzmeister@users.noreply.github.com> Date: Mon, 25 Jan 2021 15:41:18 +0100 Subject: [PATCH] Fix ingestion job config dict with null value (#226) --- utils/hsfs_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/hsfs_utils.py b/utils/hsfs_utils.py index d1deb294b7..fe557e3dab 100644 --- a/utils/hsfs_utils.py +++ b/utils/hsfs_utils.py @@ -45,7 +45,7 @@ def insert_fg(spark: SparkSession, job_conf: Dict[Any, Any]) -> None: df = get_fg_spark_df(job_conf) fg = fs.get_feature_group(name=job_conf["name"], version=job_conf["version"]) - fg.insert(df, write_options=job_conf.pop("write_options", {})) + fg.insert(df, write_options=job_conf.pop("write_options", {}) or {}) def create_td(job_conf: Dict[Any, Any]) -> None: @@ -59,8 +59,8 @@ def create_td(job_conf: Dict[Any, Any]) -> None: td = fs.get_training_dataset(name=job_conf["name"], version=job_conf["version"]) td.insert( q, - overwrite=job_conf.pop("overwrite", False), - write_options=job_conf.pop("write_options", {}), + overwrite=job_conf.pop("overwrite", False) or False, + write_options=job_conf.pop("write_options", {}) or {}, )