Skip to content

Commit

Permalink
Fix custom arabic tasks (#440)
Browse files Browse the repository at this point in the history
* removed unused params

* fix issue with task function
  • Loading branch information
clefourrier authored Dec 12, 2024
1 parent ca2fdcb commit 0135c2e
Show file tree
Hide file tree
Showing 6 changed files with 1 addition and 50 deletions.
2 changes: 0 additions & 2 deletions community_tasks/_template.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,6 @@ def __init__(
suite=["community"],
generation_size=-1,
stop_sequence=None,
output_regex=None,
frozen=False,
)


Expand Down
18 changes: 1 addition & 17 deletions community_tasks/arabic_evals.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,6 @@ def __init__(
suite=["community"],
generation_size=-1,
stop_sequence=None,
output_regex=None,
frozen=False,
trust_dataset=True,
version=0,
)
Expand Down Expand Up @@ -174,8 +172,6 @@ def __init__(
suite=["community"],
generation_size=-1,
stop_sequence=None,
output_regex=None,
frozen=False,
trust_dataset=True,
version=0,
)
Expand Down Expand Up @@ -241,8 +237,6 @@ def __init__(
suite=["community"],
generation_size=-1,
stop_sequence=None,
output_regex=None,
frozen=False,
trust_dataset=True,
version=0,
)
Expand Down Expand Up @@ -299,8 +293,6 @@ def __init__(
suite=["community"],
generation_size=-1,
stop_sequence=None,
output_regex=None,
frozen=False,
trust_dataset=True,
version=0,
)
Expand Down Expand Up @@ -361,8 +353,6 @@ def __init__(
suite=["community"],
generation_size=-1,
stop_sequence=[],
output_regex=None,
frozen=False,
trust_dataset=True,
version=0,
)
Expand Down Expand Up @@ -423,9 +413,7 @@ def arabic_exams_pfn(line, task_name: str = None):
def alghafa_pfn(line, task_name: str = None):
question = line["query"]
answer_index = int(line["label"])
# Dynamically determining the choices by excluding '__few_shots', 'query' and 'label'
choices_keys = [key for key in line.keys() if key not in ["query", "label", "__few_shots"]]
choices = [line[key] for key in choices_keys]
choices = [line[key] for key in ["sol1", "sol2", "sol3", "sol4"]]

instruction = "الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح\n\n"
query = f"{instruction}السؤال: {question}\n"
Expand Down Expand Up @@ -461,8 +449,6 @@ def __init__(
suite=["community"],
generation_size=-1,
stop_sequence=None,
output_regex=None,
frozen=False,
trust_dataset=True,
version=0,
)
Expand Down Expand Up @@ -839,8 +825,6 @@ def __init__(
suite=["community"],
generation_size=-1,
stop_sequence=None,
output_regex=None,
frozen=False,
trust_dataset=True,
version=0,
)
Expand Down
9 changes: 0 additions & 9 deletions docs/source/adding-a-custom-task.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -107,8 +107,6 @@ class CustomSubsetTask(LightevalTaskConfig):
suite=["community"],
generation_size=-1,
stop_sequence=None,
output_regex=None,
frozen=False,
)
SUBSET_TASKS = [CustomSubsetTask(name=f"mytask:{subset}", hf_subset=subset) for subset in SAMPLE_SUBSETS]
```
Expand Down Expand Up @@ -154,13 +152,6 @@ Here is a list of the parameters and their meaning:
for your generation
- `metric` (list), the metrics you want to use for your evaluation (see next
section for a detailed explanation)
- `output_regex` (str), A regex string that will be used to filter your
generation. (Generative metrics will only select tokens that are between the
first and the second sequence matched by the regex. For example, for a regex
matching `\n` and a generation `\nModel generation output\nSome other text`
the metric will only be fed with `Model generation output`)
- `frozen` (bool), for now, is set to False, but we will steadily pass all
stable tasks to True.
- `trust_dataset` (bool), set to True if you trust the dataset.


Expand Down
2 changes: 0 additions & 2 deletions docs/source/saving-and-reading-results.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -170,9 +170,7 @@ The detail file contains the following columns:
"stop_sequence": [
"Question="
],
"output_regex": null,
"num_samples": null,
"frozen": false,
"suite": [
"lighteval"
],
Expand Down
16 changes: 0 additions & 16 deletions examples/nanotron/custom_evaluation_tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -267,8 +267,6 @@ def __init__(
generation_size=40,
trust_dataset=True,
stop_sequence=None,
output_regex=None,
frozen=False,
):
super().__init__(
name=name,
Expand All @@ -282,8 +280,6 @@ def __init__(
few_shots_select=few_shots_select,
suite=suite,
generation_size=generation_size,
output_regex=output_regex,
frozen=frozen,
trust_dataset=trust_dataset,
stop_sequence=(stop_sequence if stop_sequence is not None else ["\n"]),
)
Expand Down Expand Up @@ -370,8 +366,6 @@ def __init__(
generation_size=-1,
trust_dataset=True,
stop_sequence=None,
output_regex=None,
frozen=False,
):
super().__init__(
name=name,
Expand All @@ -387,8 +381,6 @@ def __init__(
generation_size=generation_size,
trust_dataset=trust_dataset,
stop_sequence=(stop_sequence if stop_sequence is not None else ["\n"]),
output_regex=output_regex,
frozen=frozen,
)


Expand Down Expand Up @@ -487,8 +479,6 @@ def __init__(
generation_size=4,
trust_dataset=True,
stop_sequence=None,
output_regex=None,
frozen=False,
):
super().__init__(
name=name,
Expand All @@ -504,8 +494,6 @@ def __init__(
generation_size=generation_size,
trust_dataset=trust_dataset,
stop_sequence=(stop_sequence if stop_sequence is not None else ["\n"]),
output_regex=output_regex,
frozen=frozen,
)


Expand Down Expand Up @@ -623,8 +611,6 @@ def __init__(
generation_size=-1,
trust_dataset=True,
stop_sequence=None,
output_regex=None,
frozen=False,
):
super().__init__(
name=name,
Expand All @@ -640,8 +626,6 @@ def __init__(
generation_size=generation_size,
trust_dataset=trust_dataset,
stop_sequence=(stop_sequence if stop_sequence is not None else ["\n"]),
output_regex=output_regex,
frozen=frozen,
)


Expand Down
4 changes: 0 additions & 4 deletions examples/nanotron/custom_task.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,6 @@ def mmlu_anatomy(line):
generation_size=5,
metric=[Metrics.loglikelihood_acc_single_token],
stop_sequence=["\n"],
output_regex=None,
frozen=False,
),
LightevalTaskConfig(
name="mmlu:anatomy_signs",
Expand All @@ -98,7 +96,5 @@ def mmlu_anatomy(line):
generation_size=5,
metric=[Metrics.loglikelihood_acc_single_token],
stop_sequence=["\n"],
output_regex=None,
frozen=False,
),
]

0 comments on commit 0135c2e

Please sign in to comment.