updated name info assets #149
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,174 @@ | ||
| import os | ||
| import re | ||
|
|
||
| from arabic_llm_benchmark.datasets import NameInfoDataset | ||
| from arabic_llm_benchmark.models import BLOOMPetalModel | ||
| from arabic_llm_benchmark.tasks import DemographyNameInfoTask | ||
|
|
||
|
|
||
| def config(): | ||
| return { | ||
| "dataset": NameInfoDataset, | ||
| "dataset_args": {}, | ||
| "task": DemographyNameInfoTask, | ||
| "task_args": {}, | ||
| "model": BLOOMPetalModel, | ||
| "model_args": { | ||
| "api_url": os.environ["API_URL"], | ||
| "class_labels": [ | ||
| "gb", | ||
| "us", | ||
| "cl", | ||
| "fr", | ||
| "ru", | ||
| "pl", | ||
| "in", | ||
| "it", | ||
| "kr", | ||
| "gh", | ||
| "ca", | ||
| "sa", | ||
| "at", | ||
| "de", | ||
| "cn", | ||
| "br", | ||
| "dk", | ||
| "se", | ||
| "bd", | ||
| "cu", | ||
| "jp", | ||
| "be", | ||
| "es", | ||
| "co", | ||
| "id", | ||
| "iq", | ||
| "pk", | ||
| "tr", | ||
| "il", | ||
| "ch", | ||
| "ar", | ||
| "ro", | ||
| "nl", | ||
| "ps", | ||
| "ug", | ||
| "ir", | ||
| "cg", | ||
| "do", | ||
| "ee", | ||
| "tn", | ||
| "gr", | ||
| "np", | ||
| "ie", | ||
| "sy", | ||
| "hu", | ||
| "eg", | ||
| "ma", | ||
| "ve", | ||
| "ph", | ||
| "no", | ||
| "bg", | ||
| "si", | ||
| "ke", | ||
| "au", | ||
| "et", | ||
| "py", | ||
| "af", | ||
| "pt", | ||
| "th", | ||
| "bo", | ||
| "mx", | ||
| "lb", | ||
| "za", | ||
| "fi", | ||
| "hr", | ||
| "vn", | ||
| "ly", | ||
| "nz", | ||
| "qa", | ||
| "kh", | ||
| "ci", | ||
| "ng", | ||
| "sg", | ||
| "cm", | ||
| "dz", | ||
| "tz", | ||
| "ae", | ||
| "pe", | ||
| "az", | ||
| "lu", | ||
| "ec", | ||
| "cz", | ||
| "ua", | ||
| "uy", | ||
| "sd", | ||
| "ao", | ||
| "my", | ||
| "lv", | ||
| "kw", | ||
| "tw", | ||
| "bh", | ||
| "lk", | ||
| "ye", | ||
| "cr", | ||
| "jo", | ||
| "pa", | ||
| "om", | ||
| "uz", | ||
| "by", | ||
| "kz", | ||
| ], | ||
| "max_tries": 3, | ||
| }, | ||
| "general_args": { | ||
| "data_path": "data/demographic_attributes/name_info/wikidata_test.txt" | ||
| }, | ||
| } | ||
|
|
||
|
|
||
| def prompt(input_sample): | ||
| prompt_string = ( | ||
| f"You are an expert annotator who can identify the country of a person based on name.\n" | ||
| f"Label the country of the following person 'name'. Write ONLY the country code in ISO 3166-1 alpha-2 format.\n" | ||
| f"Provide only label.\n\n" | ||
| f"name: {input_sample}\n" | ||
| f"country: \n" | ||
| ) | ||
|
|
||
| return { | ||
| "prompt": prompt_string, | ||
| } | ||
|
|
||
|
|
||
| def post_process(response): | ||
| label = ( | ||
| response["outputs"] | ||
| .strip() | ||
| .replace("<s>", "") | ||
| .replace("</s>", "") | ||
| .replace("ISO 3166-1:", "") | ||
| .replace("ISO 3166-1", "") | ||
| .lower() | ||
| ) | ||
| label_list = config()["model_args"]["class_labels"] | ||
|
|
||
| # Regular expressions to catch the pattern | ||
| match = re.search(r"(country|country_code):\s*(.*)", label) | ||
| if match: | ||
| label = match.group(2).strip().lower() | ||
| if label in label_list: | ||
| label_fixed = label | ||
| elif ( | ||
| "I'm sorry, but I cannot predict the country" in label | ||
| or "I cannot predict the country" in label | ||
| ): | ||
| label_fixed = None | ||
| else: | ||
| label_fixed = None | ||
|
|
||
| # Consolidating the check for None or empty string | ||
| if not label_fixed: | ||
| label_fixed = None | ||
|
|
||
| return label_fixed | ||
|
|
||
| return label_fixed |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
|
There was a problem hiding this comment. Some cases like: {
"input": "جورج بوش الابن",
"label": "us",
"model_output": "country: US",
"filtered_output": null
}There was a problem hiding this comment. The gpt4 fewshot asset also has similar outputs, but handles it, perhaps we just need to use the same postprocessing fn? |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Some cases that we can handle easily I think:
{ "input": "جورج بوش الابن", "label": "us", "model_output": "name: جورج بوش الأب country: united states of america</s>", "filtered_output": "جورج بوش الأب country: united states of america" } { "input": "جورج ووكر بوش", "label": "us", "model_output": "country_code: US</s>", "filtered_output": null } { "input": "George W. Bush", "label": "us", "model_output": "name: George W. Bush country:us</s>", "filtered_output": "george w. bush country:us" } { "input": "أوغستو بينوشيه", "label": "cl", "model_output": "name: أوغستو بينوشيه</s>", "filtered_output": "أوغستو بينوشيه" } { "input": "Augusto Pinochet", "label": "cl", "model_output": "country:CL</s>", "filtered_output": null }Also should we explicitly set to None if none of the labels are found? (e.g. when it outputs the original name back - sample 4 above)