Ensure reference renaming is parallel-safe (#136)

* Ensure reference renaming is parallel-safe

This prefixes each reference with a token indicative of which docstring it belongs to and then relabels the text of the reference once the doctree is compiled

* Fix missing argument

* Install pdflatex in Travis

* More TeX packages

* Some sphinx require latexmk

* Move imports

* Fix comment

* Warn about #130

* Fixes to relabelling of references:

* Use label text not normalised text
* Split at first - not last
* Do not relabel non-docstring content

* Fix logic for identifying non-docscring

doc/format.rst

@@ -387,6 +387,12 @@ The sections of a function's docstring are:
  should not be required to understand it. References are numbered, starting
  from one, in the order in which they are cited.
 
+ .. warning:: **References will break tables**
+
+ Where references like [1] appear in a tables within a numpydoc
+ docstring, the table markup will be broken by numpydoc processing. See
+ `numpydoc issue #130 <https://github.com/numpy/numpydoc/issues/130>`_
+
 14. **Examples**
 
  An optional section for examples, using the `doctest

numpydoc/numpydoc.py

@@ -21,9 +21,13 @@
 import sys
 import re
 import pydoc
-import sphinx
 import inspect
 import collections
+import hashlib
+
+from docutils.nodes import citation, Text
+import sphinx
+from sphinx.addnodes import pending_xref, desc_content
 
 if sphinx.__version__ < '1.0.1':
  raise RuntimeError("Sphinx 1.0.1 or newer is required")
@@ -37,9 +41,12 @@
  sixu = lambda s: unicode(s, 'unicode_escape')
 
 
-def rename_references(app, what, name, obj, options, lines,
- reference_offset=[0]):
- # replace reference numbers so that there are no duplicates
+HASH_LEN = 12
+
+
+def rename_references(app, what, name, obj, options, lines):
+ # decorate reference numbers so that there are no duplicates
+ # these are later undecorated in the doctree, in relabel_references
  references = set()
  for line in lines:
  line = line.strip()
@@ -49,19 +56,51 @@ def rename_references(app, what, name, obj, options, lines,
  references.add(m.group(1))
 
  if references:
- for r in references:
- if r.isdigit():
- new_r = sixu("R%d") % (reference_offset[0] + int(r))
- else:
- new_r = sixu("%s%d") % (r, reference_offset[0])
+ # we use a hash to mangle the reference name to avoid invalid names
+ sha = hashlib.sha256()
+ sha.update(name.encode('utf8'))
+ prefix = 'R' + sha.hexdigest()[:HASH_LEN]
 
+ for r in references:
+ new_r = prefix + '-' + r
  for i, line in enumerate(lines):
  lines[i] = lines[i].replace(sixu('[%s]_') % r,
  sixu('[%s]_') % new_r)
  lines[i] = lines[i].replace(sixu('.. [%s]') % r,
  sixu('.. [%s]') % new_r)
 
- reference_offset[0] += len(references)
+
+def _ascend(node, cls):
+ while node and not isinstance(node, cls):
+ node = node.parent
+ return node
+
+
+def relabel_references(app, doc):
+ # Change 'hash-ref' to 'ref' in label text
+ for citation_node in doc.traverse(citation):
+ if _ascend(citation_node, desc_content) is None:
+ # no desc node in ancestry -> not in a docstring
+ # XXX: should we also somehow check it's in a References section?
+ continue
+ label_node = citation_node[0]
+ prefix, _, new_label = label_node[0].astext().partition('-')
+ assert len(prefix) == HASH_LEN + 1
+ new_text = Text(new_label)
+ label_node.replace(label_node[0], new_text)
+
+ for id in citation_node['backrefs']:
+ ref = doc.ids[id]
+ ref_text = ref[0]
+
+ # Sphinx has created pending_xref nodes with [reftext] text.
+ def matching_pending_xref(node):
+ return (isinstance(node, pending_xref) and
+ node[0].astext() == '[%s]' % ref_text)
+
+ for xref_node in ref.parent.traverse(matching_pending_xref):
+ xref_node.replace(xref_node[0], Text('[%s]' % new_text))
+ ref.replace(ref_text, new_text.copy())
 
 
 DEDUPLICATION_TAG = ' !! processed by numpydoc !!'
@@ -139,6 +178,7 @@ def setup(app, get_doc_object_=get_doc_object):
 
  app.connect('autodoc-process-docstring', mangle_docstrings)
  app.connect('autodoc-process-signature', mangle_signature)
+ app.connect('doctree-read', relabel_references)
  app.add_config_value('numpydoc_edit_link', None, False)
  app.add_config_value('numpydoc_use_plots', None, False)
  app.add_config_value('numpydoc_use_blockquotes', None, False)