Add whitespace even for inline tags

Thanks @codinguncut for suggestion. Still needs testing. re.sub is replicating xpath's normalize-space behaviour. See GH-1
TeamHG-Memex · May 26, 2017 · 6135ba6 · 6135ba6
1 parent c5cc532
commit 6135ba6
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 1 deletion.
diff --git a/html_text/html_text.py b/html_text/html_text.py
@@ -1,4 +1,6 @@
 # -*- coding: utf-8 -*-
+import re
+
 import lxml
 import lxml.etree
 from lxml.html.clean import Cleaner
@@ -41,8 +43,13 @@ def parse_html(html):
 
 def selector_to_text(sel):
     """ Convert a cleaned selector to text.
+    Almost the same as xpath normalize-space, but this also
+    adds spaces between inline elements (like <span>) which are
+    often used as block elements in html markup.
     """
-    return sel.xpath('normalize-space()').extract_first('')
+    fragments = (re.sub('\s+', ' ', x.strip())
+                 for x in sel.xpath('//text()').extract())
+    return ' '.join(x for x in fragments if x)
 
 
 def cleaned_selector(html):

diff --git a/tests/test_html_text.py b/tests/test_html_text.py
@@ -23,3 +23,8 @@ def test_extract_text_from_tree():
     html = u'<html><style>.div {}</style><body><p>Hello,   world!</body></html>'
     tree = parse_html(html)
     assert extract_text(tree) == u'Hello, world!'
+
+
+def test_inline_tags_whitespace():
+    html = u'<span>field</span><span>value</span>'
+    assert extract_text(html) == u'field value'