diff --git a/tests/replacebot_tests.py b/tests/replacebot_tests.py
index 4b22ed1610..193c952af7 100755
--- a/tests/replacebot_tests.py
+++ b/tests/replacebot_tests.py
@@ -307,8 +307,10 @@ def test_pairs_file(self):
         """Test handle_pairsfile."""
         result = replace.handle_pairsfile('non existing file')
         self.assertIsNone(result)
-        self.assertIn("No such file or directory: 'non existing file'",
-                      pywikibot.bot.ui.pop_output()[0])
+
+        msg = pywikibot.bot.ui.pop_output()[0]
+        self.assertIn('No such file or directory:', msg)
+        self.assertIn('non existing file', msg)
 
         result = replace.handle_pairsfile('tests/data/pagelist-lines.txt')
         self.assertIsNone(result)