bpo-39187: robotparser does not respect longest match

Lib/test/test_robotparser.py

@@ -224,6 +224,42 @@ class GoogleURLOrderingTest(BaseRobotTest, unittest.TestCase):
  bad = ['/folder1/anotherfile.html']
 
 
+class LongestMatchUserAgentTest(BaseRobotTest, unittest.TestCase):
+ # https://tools.ietf.org/html/draft-koster-rep-00#section-3.2
+ # The most specific rule should be used
+ robots_txt = """\
+User-agent: FooBot
+Disallow: /folder1/
+Allow: /folder1/myfile.html
+ """
+ agent = 'foobot'
+ good = ['/folder1/myfile.html']
+ bad = ['/folder1/anotherfile.html']
+
+
+class LongestMatchDefaultUserAgentTest(BaseRobotTest, unittest.TestCase):
+ # https://tools.ietf.org/html/draft-koster-rep-00#section-3.2
+ # The most specific rule should be used
+ robots_txt = """\
+User-agent: *
+Disallow: /folder1/
+Allow: /folder1/myfile.html
+ """
+ good = ['/folder1/myfile.html']
+ bad = ['/folder1/anotherfile.html']
+
+
+class EquivalentRulesTest(BaseRobotTest, unittest.TestCase):
+ # https://tools.ietf.org/html/draft-koster-rep-00#section-2.2.2
+ # The most specific rule should be used
+ robots_txt = """\
+User-agent: *
+Disallow: /folder1/
+Allow: /folder1/
+ """
+ good = ['/folder1/myfile.html', '/folder1', '/folder1']
+
+
 class DisallowQueryStringTest(BaseRobotTest, unittest.TestCase):
  # see issue #6325 for details
  robots_txt = """\
@@ -367,7 +403,7 @@ def test_basic(self):
  def test_can_fetch(self):
  self.assertTrue(self.parser.can_fetch('*', self.url('elsewhere')))
  self.assertFalse(self.parser.can_fetch('Nutch', self.base_url))
- self.assertFalse(self.parser.can_fetch('Nutch', self.url('brian')))
+ self.assertTrue(self.parser.can_fetch('Nutch', self.url('brian')))
  self.assertFalse(self.parser.can_fetch('Nutch', self.url('webstats')))
  self.assertFalse(self.parser.can_fetch('*', self.url('webstats')))
  self.assertTrue(self.parser.can_fetch('*', self.base_url))

Lib/urllib/robotparser.py

@@ -56,6 +56,12 @@ def set_url(self, url):
  self.url = url
  self.host, self.path = urllib.parse.urlparse(url)[1:3]
 
+ def _sort_rulelines(self):
+ for entry in self.entries:
+ entry.sort_rulelines()
+ if self.default_entry:
+ self.default_entry.sort_rulelines()
+
  def read(self):
  """Reads the robots.txt URL and feeds it to the parser."""
  try:
@@ -150,6 +156,7 @@ def parse(self, lines):
  self.sitemaps.append(line[1])
  if state == 2:
  self._add_entry(entry)
+ self._sort_rulelines()
 
  def can_fetch(self, useragent, url):
  """using the parsed robots.txt decide if useragent can fetch url"""
@@ -250,6 +257,16 @@ def __str__(self):
  ret.extend(map(str, self.rulelines))
  return '\n'.join(ret)
 
+ def sort_rulelines(self):
+ """Rules need to be sorted with the longest path first to ensure that
+ the longest rule is used for matching:
+ https://tools.ietf.org/html/draft-koster-rep-00#section-3.2
+
+ If an allow and disallow rule is equivalent (same path), the allow
+ SHOULD be used: https://tools.ietf.org/html/draft-koster-rep-00#section-2.2.2
+ """
+ self.rulelines.sort(key=lambda x: (len(x.path), x.allowance), reverse=True)
+
  def applies_to(self, useragent):
  """check if this entry applies to the specified agent"""
  # split the name token and make it lower case

Misc/NEWS.d/next/Library/2020-01-02-05-38-40.bpo-39187.0vXJVT.rst

@@ -0,0 +1,2 @@
+Add a sort function to respect the longest match rule as per the current internet draft: https://tools.ietf.org/html/draft-koster-rep-00#section-3.2
+The sort function also takes into account equivalent rules such that allow should be used: https://tools.ietf.org/html/draft-koster-rep-00#section-2.2.2