bpo-39187: robotparser does not respect longest match

Lib/test/test_robotparser.py

@@ -224,6 +224,42 @@ class GoogleURLOrderingTest(BaseRobotTest, unittest.TestCase):
     bad = ['/folder1/anotherfile.html']
 
 
+class LongestMatchUserAgentTest(BaseRobotTest, unittest.TestCase):
+    # https://tools.ietf.org/html/draft-koster-rep-00#section-3.2
+    # The most specific rule should be used
+    robots_txt = """\
+User-agent: FooBot
+Disallow: /folder1/
+Allow: /folder1/myfile.html
+    """
+    agent = 'foobot'
+    good = ['/folder1/myfile.html']
+    bad = ['/folder1/anotherfile.html']
+
+
+class LongestMatchDefaultUserAgentTest(BaseRobotTest, unittest.TestCase):
+    # https://tools.ietf.org/html/draft-koster-rep-00#section-3.2
+    # The most specific rule should be used
+    robots_txt = """\
+User-agent: *
+Disallow: /folder1/
+Allow: /folder1/myfile.html
+    """
+    good = ['/folder1/myfile.html']
+    bad = ['/folder1/anotherfile.html']
+
+
+class EquivalentRulesTest(BaseRobotTest, unittest.TestCase):
+    # https://tools.ietf.org/html/draft-koster-rep-00#section-2.2.2
+    # The most specific rule should be used
+    robots_txt = """\
+User-agent: *
+Disallow: /folder1/
+Allow: /folder1/
+    """
+    good = ['/folder1/myfile.html', '/folder1', '/folder1']
+
+
 class DisallowQueryStringTest(BaseRobotTest, unittest.TestCase):
     # see issue #6325 for details
     robots_txt = """\
@@ -367,7 +403,7 @@ def test_basic(self):
     def test_can_fetch(self):
         self.assertTrue(self.parser.can_fetch('*', self.url('elsewhere')))
         self.assertFalse(self.parser.can_fetch('Nutch', self.base_url))
-        self.assertFalse(self.parser.can_fetch('Nutch', self.url('brian')))
+        self.assertTrue(self.parser.can_fetch('Nutch', self.url('brian')))
         self.assertFalse(self.parser.can_fetch('Nutch', self.url('webstats')))
         self.assertFalse(self.parser.can_fetch('*', self.url('webstats')))
         self.assertTrue(self.parser.can_fetch('*', self.base_url))

Lib/urllib/robotparser.py

@@ -56,6 +56,12 @@ def set_url(self, url):
         self.url = url
         self.host, self.path = urllib.parse.urlparse(url)[1:3]
 
+    def _sort_rulelines(self):
+        for entry in self.entries:
+            entry.sort_rulelines()
+        if self.default_entry:
+            self.default_entry.sort_rulelines()
+
     def read(self):
         """Reads the robots.txt URL and feeds it to the parser."""
         try:
@@ -150,6 +156,7 @@ def parse(self, lines):
                     self.sitemaps.append(line[1])
         if state == 2:
             self._add_entry(entry)
+        self._sort_rulelines()
 
     def can_fetch(self, useragent, url):
         """using the parsed robots.txt decide if useragent can fetch url"""
@@ -250,6 +257,16 @@ def __str__(self):
         ret.extend(map(str, self.rulelines))
         return '\n'.join(ret)
 
+    def sort_rulelines(self):
+        """Rules need to be sorted with the longest path first to ensure that
+        the longest rule is used for matching:
+        https://tools.ietf.org/html/draft-koster-rep-00#section-3.2
+
+        If an allow and disallow rule is equivalent (same path), the allow
+        SHOULD be used: https://tools.ietf.org/html/draft-koster-rep-00#section-2.2.2
+        """
+        self.rulelines.sort(key=lambda x: (len(x.path), x.allowance), reverse=True)
+
     def applies_to(self, useragent):
         """check if this entry applies to the specified agent"""
         # split the name token and make it lower case

Misc/NEWS.d/next/Library/2020-01-02-05-38-40.bpo-39187.0vXJVT.rst

@@ -0,0 +1,2 @@
+Add a sort function to respect the longest match rule as per the current internet draft: https://tools.ietf.org/html/draft-koster-rep-00#section-3.2
+The sort function also takes into account equivalent rules such that allow should be used: https://tools.ietf.org/html/draft-koster-rep-00#section-2.2.2