Skip to content

Commit

Permalink
chore: lint
Browse files Browse the repository at this point in the history
  • Loading branch information
winglian committed Jan 23, 2024
1 parent 9c1c3c8 commit d5f97c3
Showing 1 changed file with 4 additions and 2 deletions.
6 changes: 4 additions & 2 deletions src/axolotl/utils/data.py
Original file line number Diff line number Diff line change
Expand Up @@ -849,7 +849,9 @@ def load_split(dataset_cfgs, _cfg):
if ds_cfg["ds_type"] == "json":
for data_file in ds_cfg["data_files"]:
data_files = {ds_cfg["split"]: data_file}
ds = load_dataset("json", data_files=data_files, split=ds_cfg["split"])
ds = load_dataset( # pylint: disable=invalid-name
"json", data_files=data_files, split=ds_cfg["split"]
)
split_datasets.insert(i, ds)
else:
ds = load_dataset( # pylint: disable=invalid-name
Expand All @@ -863,7 +865,7 @@ def load_split(dataset_cfgs, _cfg):
ds_transform_fn = load_dpo(_type, _cfg)
split_datasets[i] = data_set.map(ds_transform_fn)
else:
# If no `type` is provided, assume the dataset is already in the expected format with
# If no `type` is provided, assume the dataset is already in the expected format with
# "prompt", "chosen" and "rejected" already preprocessed
split_datasets[i] = data_set

Expand Down

0 comments on commit d5f97c3

Please sign in to comment.