Skip to content

Commit

Permalink
ensure prompt key
Browse files Browse the repository at this point in the history
Signed-off-by: Kyle Sayers <kylesayrs@gmail.com>
  • Loading branch information
kylesayrs committed Dec 20, 2024
1 parent f2b3c99 commit 5b4ee22
Showing 1 changed file with 4 additions and 4 deletions.
8 changes: 4 additions & 4 deletions src/llmcompressor/transformers/finetune/data/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -236,12 +236,12 @@ def filter_processor_args(self, dataset: DatasetType) -> DatasetType:
for key, param in signature.parameters.items()
if param.kind not in (Kind.VAR_POSITIONAL, Kind.VAR_KEYWORD)
)
logger.debug(
f"Found processor args `{processor_kwargs}`. Removing all other columns"
)
logger.debug(f"Found processor args `{processor_kwargs}`")

column_names = get_columns(dataset)
return dataset.remove_columns(list(set(column_names) - set(processor_kwargs)))
return dataset.remove_columns(
list(set(column_names) - set(processor_kwargs)) - set([self.PROMPT_KEY])
)

def tokenize(self, data: LazyRow) -> Dict[str, Any]:
# separate prompt
Expand Down

0 comments on commit 5b4ee22

Please sign in to comment.