Merge pull request #48 from jeroenseegers/reppy-compatibility

Add equal-sign to parse safe characters

src/protego.py

@@ -184,7 +184,7 @@ def _quote_pattern(self, pattern):
 
         parts = urlparse(pattern)
         pattern = self._unquote(parts.path, ignore="/*$%")
-        pattern = quote(pattern, safe="/*%")
+        pattern = quote(pattern, safe="/*%=")
 
         parts = ParseResult(
             "", "", pattern + last_char, parts.params, parts.query, parts.fragment

tests/test_protego.py

@@ -1141,6 +1141,13 @@ def test_parse_time_period(self):
         self.assertEqual(start_time, time(5, 0))
         self.assertEqual(end_time, time(6, 0))
 
+    def test_disallow_query_wildcard(self):
+        content = "User-agent: * \nDisallow: /*s="
+        rp = Protego.parse(content=content)
+        self.assertTrue(rp.can_fetch("https://www.site.local/", "*"))
+        self.assertTrue(rp.can_fetch("https://www.site.local/s/", "*"))
+        self.assertFalse(rp.can_fetch("https://www.site.local/?s=asd", "*"))
+
 
 @pytest.mark.parametrize(
     "allow,disallow,url,allowed",