[BFCL] Multi Turn Pipeline Robustness Patch (#724)

This PR adds checks to prevent unsafe function calls from being run and
potentially breaking the evaluation pipeline.

1. In GorillaFileSytem, the `xargs` method is removed. This will not
affect the dataset entries, and that function is not used in any
entries.
2. In `execute_multi_turn_func_call`, an input filter has been
implemented to prevent the execution of unsafe function calls like
`kill`, `exit`, `remove`, etc.

---------

Co-authored-by: Shishir Patil <30296397+ShishirPatil@users.noreply.github.com>

berkeley-function-call-leaderboard/bfcl/eval_checker/multi_turn_eval/func_source_code/gorilla_file_system.py

@@ -500,35 +500,6 @@ def grep(self, file_name: str, pattern: str) -> Dict[str, List[str]]:
 
         return {"error": f"grep: {file_name}: No such file or directory"}
 
-    def xargs(self, command: str, file_name: str = None):
-        """
-        Execute a command with arguments read from a file or standard input.
-
-        Args:
-            command (str): The command to execute with arguments.
-            file_name (str): [Optional] The file containing arguments. Defaults to None.
-
-        Returns:
-            output (str): The result of the command execution.
-        """
-        if file_name:
-            if file_name in self._current_dir.contents:
-                file = self._current_dir._get_item(file_name)
-                if isinstance(file, File):
-                    args = file._read().splitlines()
-                else:
-                    return {"error": f"xargs: {file_name}: Not a file"}
-            else:
-                return {"error": f"xargs: {file_name}: No such file or directory"}
-        else:
-            return {"error": f"Argument not supported"}
-
-        try:
-            result = subprocess.run([command] + args, capture_output=True, text=True)
-            return {"output": result.stdout, "error": result.stderr}
-        except Exception as e:
-            return {"error": str(e)}
-
     def du(self, human_readable: bool = False) -> Dict[str, str]:
         """
         Estimate the disk usage of a directory and its contents.

berkeley-function-call-leaderboard/bfcl/eval_checker/multi_turn_eval/multi_turn_utils.py

@@ -77,6 +77,17 @@ def execute_multi_turn_func_call(
 
         # Evaluate the function call
         try:
+            # Before calling `eval`, we need to make sure that the function call is safe
+            # We do so by checking if the function is `kill` or `exit`, etc.
+            # Extract the function name first
+            if "(" in func_call:
+                func_call = func_call.split("(")[0]
+            # Situation where the function call is a method call
+            if "." in func_call:
+                func_call = func_call.split(".")[1]
+            if func_call in ["kill", "exit", "quit", "remove", "unlink", "rmdir", "popen", "Popen", "run"]:
+                raise Exception(f"Function call {func_call} is not allowed.")
+
             func_call_result = eval(func_call)
 
             if type(func_call_result) == str: