[BFCL] Dataset and Possible Answer Fix (Live Categories) for Illegal Python Parameter Name

berkeley-function-call-leaderboard/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 All notable changes to the Berkeley Function Calling Leaderboard will be documented in this file.
 
+- [Nov 22, 2024] [#777](https://github.com/ShishirPatil/gorilla/pull/777), [#778](https://github.com/ShishirPatil/gorilla/pull/778): Fix dataset entries where the function doc contains illegal Python parameter names (such as `class`). 55 entries are affected.
 - [Nov 19, 2024] [#750](https://github.com/ShishirPatil/gorilla/pull/750): Add the following new models to the leaderboard:
   - `claude-3-5-haiku-20241022`
   - `claude-3-5-haiku-20241022-FC`

berkeley-function-call-leaderboard/data/possible_answer/BFCL_v3_live_multiple.json

@@ -241,7 +241,7 @@
 {"id": "live_multiple_240-107-1", "ground_truth": [{"search_web_tool": {"query": ["weather in India", "current weather in India", "What's the current weather in India", "What's the current weather in India?"], "num_results": ["", 3], "source": ["", "text"]}}]}
 {"id": "live_multiple_241-107-2", "ground_truth": [{"search_web_tool": {"query": ["flight prices from NYC to New Delhi"], "num_results": ["", 3], "source": ["", "text"]}}]}
 {"id": "live_multiple_242-107-3", "ground_truth": [{"tts_tool": {"content": ["If you could have dinner with any historical figure, who would you choose and why?, also genart a photo of that figure"]}}]}
-{"id": "live_multiple_243-107-4", "ground_truth": [{"search_web_tool": {"query": ["Find a recent news article about corona virus", "recent news article about corona virus","recent news corona virus"], "num_results": ["", 3], "source": ["news"]}}]}
+{"id": "live_multiple_243-107-4", "ground_truth": [{"search_web_tool": {"query": ["Find a recent news article about corona virus", "recent news article about corona virus", "recent news corona virus"], "num_results": ["", 3], "source": ["news"]}}]}
 {"id": "live_multiple_244-108-0", "ground_truth": [{"version_api.VersionApi.get_version": {}}]}
 {"id": "live_multiple_245-109-0", "ground_truth": [{"version_api.VersionApi.get_version": {}}]}
 {"id": "live_multiple_246-110-0", "ground_truth": [{"version_api.VersionApi.get_version": {}}]}
@@ -271,7 +271,7 @@
 {"id": "live_multiple_270-127-5", "ground_truth": [{"Weather_1_GetWeather": {"city": ["New York"], "date": ["2023-04-17"]}}]}
 {"id": "live_multiple_271-127-6", "ground_truth": [{"Weather_1_GetWeather": {"city": ["San Diego"], "date": ["2023-03-01"]}}]}
 {"id": "live_multiple_272-127-7", "ground_truth": [{"Weather_1_GetWeather": {"city": ["Nairobi"], "date": ["", null]}}]}
-{"id": "live_multiple_273-127-8", "ground_truth": [{"Weather_1_GetWeather": {"city": ["Phoenix","Phoenix, AZ"], "date": ["2023-04-14"]}}]}
+{"id": "live_multiple_273-127-8", "ground_truth": [{"Weather_1_GetWeather": {"city": ["Phoenix", "Phoenix, AZ"], "date": ["2023-04-14"]}}]}
 {"id": "live_multiple_274-127-9", "ground_truth": [{"Weather_1_GetWeather": {"city": ["Chicago", "Chi-town"], "date": ["2024-02-13"]}}]}
 {"id": "live_multiple_275-127-10", "ground_truth": [{"Weather_1_GetWeather": {"city": ["American Canyon"], "date": ["2023-03-12"]}}]}
 {"id": "live_multiple_276-127-11", "ground_truth": [{"Weather_1_GetWeather": {"city": ["Los Angeles"], "date": ["2023-03-09"]}}]}
@@ -484,10 +484,10 @@
 {"id": "live_multiple_483-146-8", "ground_truth": [{"Music_3_LookupMusic": {"artist": ["Sia"], "album": ["This Is Acting"], "genre": ["Pop"], "year": ["", "dontcare"]}}]}
 {"id": "live_multiple_484-146-9", "ground_truth": [{"Music_3_LookupMusic": {"artist": ["", "dontcare"], "album": ["The New Classic"], "genre": ["", "dontcare"], "year": ["2017"]}}]}
 {"id": "live_multiple_485-147-0", "ground_truth": [{"Trains_1_FindTrains": {"_from": ["New York, NY"], "to": ["Los Angeles, CA"], "date_of_journey": ["04/25/2023"]}}]}
-{"id": "live_multiple_486-147-1", "ground_truth": [{"Trains_1_GetTrainTickets": {"_from": ["New York, NY"], "to": ["Los Angeles, CA"], "date_of_journey": ["04/23/2023"], "journey_start_time": ["10:00"], "number_of_adults": [2], "trip_protection": [false], "class": ["Business"]}}]}
-{"id": "live_multiple_487-147-2", "ground_truth": [{"Trains_1_GetTrainTickets": {"_from": ["New York, NY"], "to": ["Sacramento, CA"], "date_of_journey": ["03/13/2023"], "journey_start_time": ["09:00"], "number_of_adults": [2], "trip_protection": [true], "class": ["Business"]}}]}
+{"id": "live_multiple_486-147-1", "ground_truth": [{"Trains_1_GetTrainTickets": {"_from": ["New York, NY"], "to": ["Los Angeles, CA"], "date_of_journey": ["04/23/2023"], "journey_start_time": ["10:00"], "number_of_adults": [2], "trip_protection": [false], "_class": ["Business"]}}]}
+{"id": "live_multiple_487-147-2", "ground_truth": [{"Trains_1_GetTrainTickets": {"_from": ["New York, NY"], "to": ["Sacramento, CA"], "date_of_journey": ["03/13/2023"], "journey_start_time": ["09:00"], "number_of_adults": [2], "trip_protection": [true], "_class": ["Business"]}}]}
 {"id": "live_multiple_488-147-3", "ground_truth": [{"Trains_1_FindTrains": {"_from": ["Portland, OR"], "to": ["Seattle, WA"], "date_of_journey": ["04/22/2023"]}}]}
-{"id": "live_multiple_489-147-4", "ground_truth": [{"Trains_1_GetTrainTickets": {"_from": ["New York, NY"], "to": ["Phoenix, AZ"], "date_of_journey": ["04/23/2023"], "journey_start_time": ["13:45"], "number_of_adults": [1], "trip_protection": [false], "class": ["", "Value"]}}]}
+{"id": "live_multiple_489-147-4", "ground_truth": [{"Trains_1_GetTrainTickets": {"_from": ["New York, NY"], "to": ["Phoenix, AZ"], "date_of_journey": ["04/23/2023"], "journey_start_time": ["13:45"], "number_of_adults": [1], "trip_protection": [false], "_class": ["", "Value"]}}]}
 {"id": "live_multiple_490-148-0", "ground_truth": [{"Events_3_FindEvents": {"event_type": ["Theater"], "city": ["Chicago, IL"], "date": ["2023-04-29"]}}]}
 {"id": "live_multiple_491-148-1", "ground_truth": [{"Events_3_FindEvents": {"event_type": ["Music"], "city": ["Berkeley, CA"], "date": ["2023-05-12"]}}]}
 {"id": "live_multiple_492-148-2", "ground_truth": [{"Events_3_FindEvents": {"event_type": ["Music"], "city": ["Berkeley, CA"], "date": ["2023-03-10"]}}]}
@@ -857,9 +857,9 @@
 {"id": "live_multiple_856-180-4", "ground_truth": [{"Buses_3_BuyBusTicket": {"from_city": ["Berkeley, CA"], "to_city": ["Portland, OR"], "departure_date": ["2023-05-15"], "departure_time": ["09:00"], "num_passengers": [2], "additional_luggage": [true]}}]}
 {"id": "live_multiple_857-180-5", "ground_truth": [{"Buses_3_FindBus": {"from_city": ["San Francisco, CA"], "to_city": ["Los Angeles, CA"], "departure_date": ["03/07/2023"], "num_passengers": ["", "1"], "category": ["one-stop"]}}]}
 {"id": "live_multiple_858-180-6", "ground_truth": [{"Buses_3_BuyBusTicket": {"from_city": ["San Francisco, CA"], "to_city": ["Los Angeles, CA"], "departure_date": ["2023-05-15"], "departure_time": ["14:00"], "num_passengers": [4], "additional_luggage": [true]}}]}
-{"id": "live_multiple_859-181-0", "ground_truth": [{"Trains_1_GetTrainTickets": {"_from": ["New York, NY"], "to": ["Washington, DC"], "date_of_journey": ["10/11/2023"], "journey_start_time": ["09:00"], "number_of_adults": [1, ""], "trip_protection": [true], "class": ["Business"]}}]}
+{"id": "live_multiple_859-181-0", "ground_truth": [{"Trains_1_GetTrainTickets": {"_from": ["New York, NY"], "to": ["Washington, DC"], "date_of_journey": ["10/11/2023"], "journey_start_time": ["09:00"], "number_of_adults": [1, ""], "trip_protection": [true], "_class": ["Business"]}}]}
 {"id": "live_multiple_860-181-1", "ground_truth": [{"Trains_1_FindTrains": {"_from": ["San Francisco, CA"], "to": ["Los Angeles, CA"], "date_of_journey": ["2023-04-15"]}}]}
-{"id": "live_multiple_861-181-2", "ground_truth": [{"Trains_1_GetTrainTickets": {"_from": ["Sacramento, CA"], "to": ["Portland, OR"], "date_of_journey": ["04/22/2023"], "journey_start_time": ["10:00"], "number_of_adults": [1, ""], "trip_protection": [false, ""], "class": ["Value", ""]}}]}
+{"id": "live_multiple_861-181-2", "ground_truth": [{"Trains_1_GetTrainTickets": {"_from": ["Sacramento, CA"], "to": ["Portland, OR"], "date_of_journey": ["04/22/2023"], "journey_start_time": ["10:00"], "number_of_adults": [1, ""], "trip_protection": [false, ""], "_class": ["Value", ""]}}]}
 {"id": "live_multiple_862-181-3", "ground_truth": [{"Trains_1_FindTrains": {"_from": ["New York, NY"], "to": ["Los Angeles, CA"], "date_of_journey": ["05/15/2023"], "journey_start_time": ["09:00"]}}]}
 {"id": "live_multiple_863-182-0", "ground_truth": [{"Movies_1_FindMovies": {"location": ["Orinda, CA"], "theater_name": ["Orinda Theatre"], "genre": ["", "dontcare"], "show_type": ["regular"]}}]}
 {"id": "live_multiple_864-182-1", "ground_truth": [{"Movies_1_FindMovies": {"location": ["Los Angeles, CA"], "theater_name": ["", "dontcare"], "genre": ["", "dontcare"], "show_type": ["regular"]}}]}
@@ -903,8 +903,8 @@
 {"id": "live_multiple_902-185-12", "ground_truth": [{"Payment_1_MakePayment": {"payment_method": ["debit card"], "amount": [25.0], "receiver": ["Alex"], "private_visibility": [true]}}]}
 {"id": "live_multiple_903-186-0", "ground_truth": [{"Trains_1_FindTrains": {"_from": ["Phoenix, AZ"], "to": ["Los Angeles, CA"], "date_of_journey": ["04/25/2023"]}}]}
 {"id": "live_multiple_904-186-1", "ground_truth": [{"Trains_1_FindTrains": {"_from": ["New York, NY"], "to": ["Los Angeles, CA"], "date_of_journey": ["03/04/2023"]}}]}
-{"id": "live_multiple_905-186-2", "ground_truth": [{"Trains_1_GetTrainTickets": {"_from": ["Portland, OR"], "to": ["Seattle, WA"], "date_of_journey": ["2023-05-15"], "journey_start_time": ["dontcare"], "number_of_adults": [4, ""], "trip_protection": [true], "class": ["Business"]}}]}
-{"id": "live_multiple_906-186-3", "ground_truth": [{"Trains_1_GetTrainTickets": {"_from": ["New York, NY"], "to": ["Los Angeles, CA"], "date_of_journey": ["2023-05-05"], "journey_start_time": ["15:00"], "number_of_adults": [2, ""], "trip_protection": [true], "class": ["Business"]}}]}
+{"id": "live_multiple_905-186-2", "ground_truth": [{"Trains_1_GetTrainTickets": {"_from": ["Portland, OR"], "to": ["Seattle, WA"], "date_of_journey": ["2023-05-15"], "journey_start_time": ["dontcare"], "number_of_adults": [4, ""], "trip_protection": [true], "_class": ["Business"]}}]}
+{"id": "live_multiple_906-186-3", "ground_truth": [{"Trains_1_GetTrainTickets": {"_from": ["New York, NY"], "to": ["Los Angeles, CA"], "date_of_journey": ["2023-05-05"], "journey_start_time": ["15:00"], "number_of_adults": [2, ""], "trip_protection": [true], "_class": ["Business"]}}]}
 {"id": "live_multiple_907-186-4", "ground_truth": [{"Trains_1_FindTrains": {"_from": ["Philadelphia, PA"], "to": ["New York, NY"], "date_of_journey": ["03/08/2023"]}}]}
 {"id": "live_multiple_908-187-0", "ground_truth": [{"Travel_1_FindAttractions": {"location": ["New York, NY"], "free_entry": ["True"], "category": ["", "dontcare"], "good_for_kids": ["True"]}}]}
 {"id": "live_multiple_909-188-0", "ground_truth": [{"youtube.check_videos": {"title": ["The Secrets of the Taj Mahal"], "publisher": ["TravelHistory"], "tags": [["Taj Mahal", "India", "architecture"]]}}]}

berkeley-function-call-leaderboard/data/possible_answer/BFCL_v3_live_parallel.json

@@ -1,7 +1,7 @@
 {"id": "live_parallel_0-0-0", "ground_truth": [{"get_current_weather": {"location": ["Beijing, China"], "unit": ["", "fahrenheit"]}}, {"get_current_weather": {"location": ["Shanghai, China"], "unit": ["", "fahrenheit"]}}]}
 {"id": "live_parallel_1-0-1", "ground_truth": [{"get_current_weather": {"location": ["Boston, MA"], "unit": ["", "fahrenheit"]}}, {"get_current_weather": {"location": ["San Francisco, CA"], "unit": ["", "fahrenheit"]}}]}
 {"id": "live_parallel_2-0-2", "ground_truth": [{"get_current_weather": {"location": ["Boston, MA"], "unit": ["", "fahrenheit"]}}, {"get_current_weather": {"location": ["San Francisco, CA"], "unit": ["", "fahrenheit"]}}]}
-{"id": "live_parallel_3-0-3", "ground_truth": [{"get_current_weather": {"location": ["Canc\u00fan, QR","Canc\\u00fan, QR"], "unit": ["", "fahrenheit"]}}, {"get_current_weather": {"location": ["Playa del Carmen, QR"], "unit": ["", "fahrenheit"]}}, {"get_current_weather": {"location": ["Tulum, QR"], "unit": ["", "fahrenheit"]}}]}
+{"id": "live_parallel_3-0-3", "ground_truth": [{"get_current_weather": {"location": ["Canc\u00fan, QR", "Canc\\u00fan, QR"], "unit": ["", "fahrenheit"]}}, {"get_current_weather": {"location": ["Playa del Carmen, QR"], "unit": ["", "fahrenheit"]}}, {"get_current_weather": {"location": ["Tulum, QR"], "unit": ["", "fahrenheit"]}}]}
 {"id": "live_parallel_4-1-0", "ground_truth": [{"get_current_weather": {"location": ["Boston, USA"], "url": ["", "https://api.open-meteo.com/v1/forecast"]}}, {"get_current_weather": {"location": ["San Francisco, USA"], "url": ["", "https://api.open-meteo.com/v1/forecast"]}}]}
 {"id": "live_parallel_5-2-0", "ground_truth": [{"get_current_weather": {"location": ["Boston, MA"], "unit": ["", "fahrenheit"], "url": ["", "https://api.open-meteo.com/v1/forecast"]}}, {"get_current_weather": {"location": ["San Francisco, CA"], "unit": ["", "fahrenheit"], "url": ["", "https://api.open-meteo.com/v1/forecast"]}}]}
 {"id": "live_parallel_6-3-0", "ground_truth": [{"get_snow_report": {"location": ["Paris, France"], "unit": ["", "fahrenheit"]}}, {"get_snow_report": {"location": ["Bordeaux, France"], "unit": ["", "fahrenheit"]}}]}

berkeley-function-call-leaderboard/data/possible_answer/BFCL_v3_live_simple.json

@@ -143,15 +143,15 @@
 {"id": "live_simple_142-94-1", "ground_truth": [{"cmd_controller.execute": {"command": ["dir Desktop"], "unit": ["", "N/A"]}}]}
 {"id": "live_simple_143-95-0", "ground_truth": [{"cmd_controller.execute": {"command": ["docker ps"], "unit": ["", "N/A"]}}]}
 {"id": "live_simple_144-95-1", "ground_truth": [{"cmd_controller.execute": {"command": ["taskkill /F /IM firefox.exe", "taskkill /IM firefox.exe /F", "taskkill /IM firefox.exe"], "unit": ["", "N/A"]}}]}
-{"id": "live_simple_145-95-2", "ground_truth": [{"cmd_controller.execute": {"command": ["dir C:\\","dir C:\\\\"], "unit": ["", "N/A"]}}]}
+{"id": "live_simple_145-95-2", "ground_truth": [{"cmd_controller.execute": {"command": ["dir C:\\", "dir C:\\\\"], "unit": ["", "N/A"]}}]}
 {"id": "live_simple_146-95-3", "ground_truth": [{"cmd_controller.execute": {"command": ["start https://huggingface.co"], "unit": ["", "N/A"]}}]}
 {"id": "live_simple_147-95-4", "ground_truth": [{"cmd_controller.execute": {"command": ["taskkill /F /IM timer.exe"], "unit": ["", "N/A"]}}]}
-{"id": "live_simple_148-95-5", "ground_truth": [{"cmd_controller.execute": {"command": ["dir C:\\","dir C:\\\\"], "unit": ["", "N/A"]}}]}
+{"id": "live_simple_148-95-5", "ground_truth": [{"cmd_controller.execute": {"command": ["dir C:\\", "dir C:\\\\"], "unit": ["", "N/A"]}}]}
 {"id": "live_simple_149-95-6", "ground_truth": [{"cmd_controller.execute": {"command": ["echo hi"], "unit": ["", "N/A"]}}]}
 {"id": "live_simple_150-95-7", "ground_truth": [{"cmd_controller.execute": {"command": ["shutdown /s /t 0"], "unit": ["", "N/A"]}}]}
 {"id": "live_simple_151-95-8", "ground_truth": [{"cmd_controller.execute": {"command": ["netsh wlan disconnect"], "unit": ["", "N/A"]}}]}
 {"id": "live_simple_152-95-9", "ground_truth": [{"cmd_controller.execute": {"command": ["dir D:\\\\ && echo testing.txt"], "unit": ["", "N/A"]}}]}
-{"id": "live_simple_153-95-10", "ground_truth": [{"cmd_controller.execute": {"command": ["del \"%userprofile%\\Desktop\\putty.lnk\"","del \\\"%userprofile%\\\\Desktop\\\\putty.lnk\\"], "unit": ["", "N/A"]}}]}
+{"id": "live_simple_153-95-10", "ground_truth": [{"cmd_controller.execute": {"command": ["del \"%userprofile%\\Desktop\\putty.lnk\"", "del \\\"%userprofile%\\\\Desktop\\\\putty.lnk\\"], "unit": ["", "N/A"]}}]}
 {"id": "live_simple_154-95-11", "ground_truth": [{"cmd_controller.execute": {"command": ["start https://www.youtube.com/watch?v=dQw4w9WgXcQ", "start https://www.youtube.com/watch?v=dQw4w9WgXcQ\\"], "unit": ["", "N/A"]}}]}
 {"id": "live_simple_155-95-12", "ground_truth": [{"cmd_controller.execute": {"command": ["date /t", "echo %date%"], "unit": ["", "N/A"]}}]}
 {"id": "live_simple_156-95-13", "ground_truth": [{"cmd_controller.execute": {"command": ["dir C:\\", "dir C:\\\\"], "unit": ["", "N/A"]}}]}
@@ -165,10 +165,10 @@
 {"id": "live_simple_164-97-0", "ground_truth": [{"get_copyright_info": {"copyright_content": ["Coca-Cola logo"], "copyright_holder": ["The Coca-Cola Company"], "confidence_score": [100.0]}}]}
 {"id": "live_simple_165-98-0", "ground_truth": [{"extractor.extract_information": {"data": [[{"name": ["\u674e\u96f7"], "age": [18]}, {"name": ["\u674e\u4e3d"], "age": [21]}]], "schema": ["personal_info"]}}]}
 {"id": "live_simple_166-99-0", "ground_truth": [{"cmd_controller.execute": {"command": ["echo %time%"]}}]}
-{"id": "live_simple_167-99-1", "ground_truth": [{"cmd_controller.execute": {"command": ["tasklist /FI \"IMAGENAME eq Code.exe\"","tasklist /FI \\\"IMAGENAME eq Code.exe\\\""]}}]}
+{"id": "live_simple_167-99-1", "ground_truth": [{"cmd_controller.execute": {"command": ["tasklist /FI \"IMAGENAME eq Code.exe\"", "tasklist /FI \\\"IMAGENAME eq Code.exe\\\""]}}]}
 {"id": "live_simple_168-99-2", "ground_truth": [{"cmd_controller.execute": {"command": ["docker start"]}}]}
 {"id": "live_simple_169-99-3", "ground_truth": [{"cmd_controller.execute": {"command": ["d:/playground/pc_contoller/env/Scripts/python.exe d:/playground/pc_contoller/test.py"]}}]}
-{"id": "live_simple_170-99-4", "ground_truth": [{"cmd_controller.execute": {"command": ["dir C:\\","dir C:\\\\"]}}]}
+{"id": "live_simple_170-99-4", "ground_truth": [{"cmd_controller.execute": {"command": ["dir C:\\", "dir C:\\\\"]}}]}
 {"id": "live_simple_171-99-5", "ground_truth": [{"cmd_controller.execute": {"command": ["echo %date%"]}}]}
 {"id": "live_simple_172-99-6", "ground_truth": [{"cmd_controller.execute": {"command": ["docker ps"]}}]}
 {"id": "live_simple_173-99-7", "ground_truth": [{"cmd_controller.execute": {"command": ["start calc"]}}]}
@@ -224,7 +224,7 @@
 {"id": "live_simple_223-117-15", "ground_truth": [{"Movies_3_FindMovies": {"directed_by": ["", "dontcare"], "genre": ["Fantasy"], "cast": ["", "dontcare"]}}]}
 {"id": "live_simple_224-117-16", "ground_truth": [{"Movies_3_FindMovies": {"directed_by": ["Tim Burton"], "genre": ["Offbeat"], "cast": ["Johnny Depp"]}}]}
 {"id": "live_simple_225-117-17", "ground_truth": [{"Movies_3_FindMovies": {"directed_by": ["Kirill Mikhanovsky"], "genre": ["Comedy-drama"], "cast": ["", "dontcare"]}}]}
-{"id": "live_simple_226-118-0", "ground_truth": [{"text_to_speech.convert": {"text": ["I am a pretty girl","Soy una chica bonita"], "language": ["es-ES"], "gender": ["", "female"], "format": ["", "mp3"], "speed": ["", 1.0]}}]}
+{"id": "live_simple_226-118-0", "ground_truth": [{"text_to_speech.convert": {"text": ["I am a pretty girl", "Soy una chica bonita"], "language": ["es-ES"], "gender": ["", "female"], "format": ["", "mp3"], "speed": ["", 1.0]}}]}
 {"id": "live_simple_227-118-1", "ground_truth": [{"text_to_speech.convert": {"text": ["\u6211\u7231\u5b66\u4e60"], "language": ["zh-CN"], "gender": ["male"], "format": ["wav"], "speed": ["", 1.0]}}]}
 {"id": "live_simple_228-119-0", "ground_truth": [{"text_to_speech.convert": {"text": ["I am a pretty girl"], "language": ["", "en-US"], "gender": ["", "female"], "speed": ["", 1.0]}}]}
 {"id": "live_simple_229-120-0", "ground_truth": [{"requests.get": {"anchor": ["user"]}}]}