diff --git a/src/huggingface_hub/hf_file_system.py b/src/huggingface_hub/hf_file_system.py
index 424923db25..bfdbb9bd59 100644
--- a/src/huggingface_hub/hf_file_system.py
+++ b/src/huggingface_hub/hf_file_system.py
@@ -244,9 +244,8 @@ def rm(
         **kwargs,
     ) -> None:
         resolved_path = self.resolve_path(path, revision=revision)
-        root_path = REPO_TYPES_URL_PREFIXES.get(resolved_path.repo_type, "") + resolved_path.repo_id
         paths = self.expand_path(path, recursive=recursive, maxdepth=maxdepth, revision=revision)
-        paths_in_repo = [path[len(root_path) + 1 :] for path in paths if not self.isdir(path)]
+        paths_in_repo = [self.resolve_path(path).path_in_repo for path in paths if not self.isdir(path)]
         operations = [CommitOperationDelete(path_in_repo=path_in_repo) for path_in_repo in paths_in_repo]
         commit_message = f"Delete {path} "
         commit_message += "recursively " if recursive else ""
diff --git a/tests/test_hf_file_system.py b/tests/test_hf_file_system.py
index 40ffc4adc0..00e389a24b 100644
--- a/tests/test_hf_file_system.py
+++ b/tests/test_hf_file_system.py
@@ -141,10 +141,14 @@ def test_file_type(self):
     def test_remove_file(self):
         self.hffs.rm_file(self.hf_path + "/data/text_data.txt")
         self.assertEqual(self.hffs.glob(self.hf_path + "/data/*"), [self.hf_path + "/data/binary_data.bin"])
+        self.hffs.rm_file(self.hf_path + "@refs/pr/1" + "/data/binary_data_for_pr.bin")
+        self.assertEqual(self.hffs.glob(self.hf_path + "@refs/pr/1" + "/data/*"), [])
 
     def test_remove_directory(self):
         self.hffs.rm(self.hf_path + "/data", recursive=True)
         self.assertNotIn(self.hf_path + "/data", self.hffs.ls(self.hf_path))
+        self.hffs.rm(self.hf_path + "@refs/pr/1" + "/data", recursive=True)
+        self.assertNotIn(self.hf_path + "@refs/pr/1" + "/data", self.hffs.ls(self.hf_path))
 
     def test_read_file(self):
         with self.hffs.open(self.hf_path + "/data/text_data.txt", "r") as f: