From cf0df5080d8f7a9cdc1d22741d8bd3ea0f95e24f Mon Sep 17 00:00:00 2001
From: Sebastian Berg <sebastianb@nvidia.com>
Date: Mon, 29 May 2023 10:36:50 +0200
Subject: [PATCH 01/10] Implement --doctest-plus-generate-diff to fix existing
 docs

---
 README.rst                     |  50 +++++++++
 pytest_doctestplus/newhooks.py |   6 ++
 pytest_doctestplus/plugin.py   | 182 ++++++++++++++++++++++++++++++++-
 3 files changed, 236 insertions(+), 2 deletions(-)
 create mode 100644 pytest_doctestplus/newhooks.py

diff --git a/README.rst b/README.rst
index d59148c..b0faf60 100644
--- a/README.rst
+++ b/README.rst
@@ -41,6 +41,10 @@ providing the following features:
 * optional inclusion of ``*.rst`` files for doctests (see `Setup and Configuration`_)
 * optional inclusion of doctests in docstrings of Numpy ufuncs
 
+Further, `pytest-doctestplus` supports editing files to fix incorrect docstrings
+(See `Fixing Existing Docstrings`).
+
+.. _pytest-remotedata: https://github.com/astropy/pytest-remotedata
 
 Installation
 ------------
@@ -374,6 +378,52 @@ running the doctests with sphinx is not supported. To do this, add
 ``conf.py`` file.
 
 
+Fixing Existing Docstrings
+--------------------------
+The plugin has basic support to fix docstrings, this can be enabled by
+running `pytest` with `--doctest-plus-generate-diff`.
+Without further options, this will print out a diff and a list of files that
+would be modified.  Using `--doctest-plus-generate-diff=overwrite` will
+modify the files in-place, so it is recommended to run the check first to
+verify the paths.
+You may wish to use e.g. `git commit -p` to review changes manually.
+
+The current diff generation is not very smart, so it does not account for
+existing `...`.  By default a diff is only generated for *failing* doctests.
+
+In general, a mass edit may wish to focus on a specific change and
+possibly include passing tests.  So you can hook into the behavior by
+adding a hook to your `conftest.py`:
+```python
+@pytest.hookimpl
+def pytest_doctestplus_diffhook(info):
+    info["use"] = True  # Overwrite all results (even successes)
+    if info["fileno"] is None:
+        # E.g. NumPy has C docstrings that cannot be found, we can add custom
+        # logic here to try and find these:
+        info["filename"] = ...
+        info["lineno"] = ...
+```
+Where `info` is a dictionary containing the following items:
+* `use`: `True` or `False` signalling whether to apply the diff.  This is
+  set to `False` if a doctest succeeded and `True` if the doctest failed.
+* `name`: The name of the test (e.g. the function being documented)
+* `filename`: The file that contains the test (this can be wrong in certain
+  situation and in that case `test_lineno` will be wrong as well).
+* `source`: The source code that was executed for this test
+* `test_lineno`: The line of code where the example block (or function) starts.
+  In some cases, the test file cannot be found and the lineno will be `None`,
+  you can manually try to fix these.
+* `example_lineno`: The line number of the example snippet (individual `>>>`).
+* `want`: The current documentation.
+* `got`: The result of executing the example.
+
+You can modify the dictionary in-place to modify the behavior.
+
+Please note that we assume that this API will be used only occasionally and
+reserve the right to change it at any time.
+
+
 Development Status
 ------------------
 
diff --git a/pytest_doctestplus/newhooks.py b/pytest_doctestplus/newhooks.py
new file mode 100644
index 0000000..94177ec
--- /dev/null
+++ b/pytest_doctestplus/newhooks.py
@@ -0,0 +1,6 @@
+# Copyright (c) 2023, Scientific Python Developers, NVIDIA see LICENSE.rst
+# SPDX-License-Identifier: BSD-3-Clause
+
+
+def pytest_doctestplus_diffhook(info):
+    """ called when a diff would be generated normally. """
\ No newline at end of file
diff --git a/pytest_doctestplus/plugin.py b/pytest_doctestplus/plugin.py
index 828a655..1d1b016 100644
--- a/pytest_doctestplus/plugin.py
+++ b/pytest_doctestplus/plugin.py
@@ -1,4 +1,6 @@
-# Licensed under a 3-clause BSD style license - see LICENSE.rst
+# Copyright (c) 2023, Scientific Python Developers, NVIDIA see LICENSE.rst
+# SPDX-License-Identifier: BSD-3-Clause
+
 """
 This plugin provides advanced doctest support and enables the testing of .rst
 files.
@@ -8,8 +10,11 @@
 import os
 import re
 import sys
+import tempfile
 import warnings
+from collections import defaultdict
 from pathlib import Path
+import subprocess
 from textwrap import indent
 from unittest import SkipTest
 
@@ -119,6 +124,18 @@ def pytest_addoption(parser):
     parser.addoption("--doctest-only", action="store_true",
                      help="Test only doctests. Implies usage of doctest-plus.")
 
+    parser.addoption("--doctest-plus-generate-diff",
+                  help="Generate a diff for where expected output and real output "
+                  "differ.  The diff is printed to stdout if not using "
+                  "`--doctest-plus-generate-diff=overwrite` which causes editing of "
+                  "the original files.\n"
+                  "NOTE: Unless an in-pace build is picked "
+                  "up, python file paths may point to unexpected places. "
+                  "If inplace is not used, will create a temporary folder and "
+                  "use `git diff -p` to generate a diff.",
+                  choices=["diff", "overwrite"],
+                  action="store", nargs="?", default=False, const="diff")
+
     parser.addini("text_file_format",
                   "Default format for docs. "
                   "This is no longer recommended, use --doctest-glob instead.")
@@ -160,6 +177,11 @@ def pytest_addoption(parser):
                   default=[])
 
 
+def pytest_addhooks(pluginmanager):
+    from pytest_doctestplus import newhooks
+    method = pluginmanager.add_hookspecs(newhooks)
+
+
 def get_optionflags(parent):
     optionflags_str = parent.config.getini('doctest_optionflags')
     flag_int = 0
@@ -211,6 +233,10 @@ def pytest_configure(config):
     for ext, chars in ext_comment_pairs:
         comment_characters[ext] = chars
 
+    # Fetch the global hook function:
+    global doctestplus_diffhook
+    doctestplus_diffhook = config.hook.pytest_doctestplus_diffhook
+
     class DocTestModulePlus(doctest_plugin.DoctestModule):
         # pytest 2.4.0 defines "collect".  Prior to that, it defined
         # "runtest".  The "collect" approach is better, because we can
@@ -269,6 +295,7 @@ def collect(self):
                 checker=OutputChecker(),
                 # Helper disables continue-on-failure when debugging is enabled
                 continue_on_failure=_get_continue_on_failure(config),
+                generate_diff=config.option.doctest_plus_generate_diff,
             )
 
             for test in finder.find(module):
@@ -333,6 +360,7 @@ def collect(self):
             runner = DebugRunnerPlus(
                 verbose=False, optionflags=optionflags, checker=OutputChecker(),
                 continue_on_failure=_get_continue_on_failure(self.config),
+                generate_diff=self.config.option.doctest_plus_generate_diff,
             )
 
             parser = DocTestParserPlus()
@@ -736,8 +764,125 @@ def test_filter(test):
         return tests
 
 
+def write_modified_file(fname, new_fname, changes):
+    # Sort in reversed order to edit the lines:
+    bad_tests = []
+    changes.sort(key=lambda x: (x["test_lineno"], x["example_lineno"]),
+                    reverse=True)
+
+    with open(fname, "r") as f:
+        text = f.readlines()
+
+    for change in changes:
+        if change["test_lineno"] is None:
+            bad_tests.append(change["name"])
+            continue
+        lineno = change["test_lineno"] + change["example_lineno"] + 1
+
+        indentation = " " * change["nindent"]
+        want = indent(change["want"], indentation, lambda x: True)
+        # Replace fully blank lines with the required `<BLANKLINE>`
+        # (May need to do this also if line contains only whitespace)
+        got = change["got"].replace("\n\n", "\n<BLANKLINE>\n")
+        got = indent(got, indentation, lambda x: True)
+
+        text[lineno:lineno+want.count("\n")] = [got]
+
+    with open(new_fname, "w") as f:
+        f.write("".join(text))
+
+    return bad_tests
+
+
+def pytest_terminal_summary(terminalreporter, exitstatus, config):
+    changesets = DebugRunnerPlus._changesets
+    diff_mode = DebugRunnerPlus._generate_diff
+    all_bad_tests = []
+    if not diff_mode:
+        return  # we do not report or apply diffs
+
+    if diff_mode != "overwrite":
+        # In this mode, we write a corrected file to a temporary folder in
+        # order to compare them (rather than modifying the file).
+        terminalreporter.section("Reporting DoctestPlus Diffs")
+        if not changesets:
+            terminalreporter.write_line("No doc changes to show")
+            return
+
+        # Strip away the common part of the path to make it a bit clearner...
+        common_path = os.path.commonpath(changesets.keys())
+        if not os.path.isdir(common_path):
+            common_path = os.path.split(common_path)[0]
+
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            for fname, changes in changesets.items():
+                # Create a new filename and ensure the path exists (in the
+                # temporary directory).
+                new_fname = fname.replace(common_path, tmpdirname)
+                os.makedirs(os.path.split(new_fname)[0], exist_ok=True)
+
+                bad_tests = write_modified_file(fname, new_fname, changes)
+                all_bad_tests.extend(bad_tests)
+
+                # git diff returns 1 to signal changes, so just ignore the
+                # exit status:
+                with subprocess.Popen(
+                        ["git", "diff", "-p", "--no-index", fname, new_fname],
+                        stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) as p:
+                    p.wait()
+                    # Diff should be fine, but write error if not:
+                    diff = p.stderr.read()
+                    diff += p.stdout.read()
+
+                    # hide the temporary directory (cleaning up anyway):
+                    if not os.path.isabs(common_path):
+                        diff = diff.replace(tmpdirname, "/" + common_path)
+                    else:
+                        # diff seems to not include extra /
+                        diff = diff.replace(tmpdirname, common_path)
+                    terminalreporter.write(diff)
+                    terminalreporter.write_line(f"{tmpdirname}, {common_path}")
+
+                terminalreporter.section("Files with modifications", "-")
+                terminalreporter.write_line(
+                    "The following files would be overwritten with "
+                    "`--doctest-plus-generate-diff=overwrite`:")
+                for fname in changesets:
+                    terminalreporter.write_line(f"    {fname}")
+                terminalreporter.write_line(
+                    "make sure these file paths are correct before calling it!")
+    else:
+        # We are in overwrite mode so will write the modified version directly
+        # back into the same file and only report which files were changed.
+        terminalreporter.section("DoctestPlus Fixing File Docs")
+        if not changesets:
+            terminalreporter.write_line("No doc changes to apply")
+            return
+        terminalreporter.write_line("Applied fix to the following files:")
+        for fname, changes in changesets.items():
+            bad_tests = write_modified_file(fname, fname, changes)
+            all_bad_tests.extend(bad_tests)
+            terminalreporter.write_line(f"    {fname}")
+
+    if all_bad_tests:
+        terminalreporter.section("Broken Linenumbers", "-")
+        terminalreporter.write_line(
+            "Doctestplus was unable to fix the following tests "
+            "(their source is hidden or `__module__` overridden?)")
+        for bad_test in all_bad_tests:
+            terminalreporter.write_line(f"    {bad_test}")
+        terminalreporter.write_line(
+            "You can implementing a hook function to fix this (see README).")
+
+
 class DebugRunnerPlus(doctest.DebugRunner):
-    def __init__(self, checker=None, verbose=None, optionflags=0, continue_on_failure=True):
+    _changesets = defaultdict(lambda: [])
+    _generate_diff = False
+
+    def __init__(self, checker=None, verbose=None, optionflags=0, continue_on_failure=True, generate_diff=None):
+        # generated_diff is False, "diff", or "inplace" (only need truthiness)
+        DebugRunnerPlus._generate_diff = generate_diff
+
         super().__init__(checker=checker, verbose=verbose, optionflags=optionflags)
         self.continue_on_failure = continue_on_failure
 
@@ -757,3 +902,36 @@ def report_unexpected_exception(self, out, test, example, exc_info):
             out.append(failure)
         else:
             raise failure
+
+    def track_diff(self, use, out, test, example, got):
+        if example.want == got:
+            return
+
+        info = dict(use=use, name=test.name, filename=test.filename,
+                    source=example.source, nindent=example.indent,
+                    want=example.want, got=got, test_lineno=test.lineno,
+                    example_lineno=example.lineno)
+        doctestplus_diffhook(info=info)
+        if not info["use"]:
+            return
+        name = info["name"]
+        filename = info["filename"]
+        source = info["source"]
+        test_lineno = info["test_lineno"]
+        example_lineno = info["example_lineno"]
+        want = info["want"]
+        got = info["got"]
+
+        self._changesets[filename].append(info)
+
+    def report_success(self, out, test, example, got):
+        if self._generate_diff is None:
+            return super().report_success(out, test, example, got)
+
+        return self.track_diff(True, out, test, example, got)
+
+    def report_failure(self, out, test, example, got):
+        if self._generate_diff is None:
+            return super().report_success(out, test, example, got)
+
+        return self.track_diff(False, out, test, example, got)

From d8c11ef395a8aae2a1f82c9be277fb7671e8a9b2 Mon Sep 17 00:00:00 2001
From: Sebastian Berg <sebastianb@nvidia.com>
Date: Fri, 6 Oct 2023 16:19:36 +0200
Subject: [PATCH 02/10] Revert license adjustments as requested

---
 pytest_doctestplus/newhooks.py | 3 +--
 pytest_doctestplus/plugin.py   | 3 +--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/pytest_doctestplus/newhooks.py b/pytest_doctestplus/newhooks.py
index 94177ec..09a96c1 100644
--- a/pytest_doctestplus/newhooks.py
+++ b/pytest_doctestplus/newhooks.py
@@ -1,5 +1,4 @@
-# Copyright (c) 2023, Scientific Python Developers, NVIDIA see LICENSE.rst
-# SPDX-License-Identifier: BSD-3-Clause
+# Licensed under a 3-clause BSD style license - see LICENSE.rst
 
 
 def pytest_doctestplus_diffhook(info):
diff --git a/pytest_doctestplus/plugin.py b/pytest_doctestplus/plugin.py
index 1d1b016..3840b33 100644
--- a/pytest_doctestplus/plugin.py
+++ b/pytest_doctestplus/plugin.py
@@ -1,5 +1,4 @@
-# Copyright (c) 2023, Scientific Python Developers, NVIDIA see LICENSE.rst
-# SPDX-License-Identifier: BSD-3-Clause
+# Licensed under a 3-clause BSD style license - see LICENSE.rst
 
 """
 This plugin provides advanced doctest support and enables the testing of .rst

From 5aa6fb442ee4155adc20f952bf18f71cfdd3e2bd Mon Sep 17 00:00:00 2001
From: Sebastian Berg <sebastianb@nvidia.com>
Date: Fri, 6 Oct 2023 17:13:05 +0200
Subject: [PATCH 03/10] BUG: Fix issue with report_success/report_failure

Had tested the diff creation, but missed testing the normal operation...
---
 pytest_doctestplus/plugin.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/pytest_doctestplus/plugin.py b/pytest_doctestplus/plugin.py
index 3840b33..ffb53e5 100644
--- a/pytest_doctestplus/plugin.py
+++ b/pytest_doctestplus/plugin.py
@@ -878,7 +878,7 @@ class DebugRunnerPlus(doctest.DebugRunner):
     _changesets = defaultdict(lambda: [])
     _generate_diff = False
 
-    def __init__(self, checker=None, verbose=None, optionflags=0, continue_on_failure=True, generate_diff=None):
+    def __init__(self, checker=None, verbose=None, optionflags=0, continue_on_failure=True, generate_diff=False):
         # generated_diff is False, "diff", or "inplace" (only need truthiness)
         DebugRunnerPlus._generate_diff = generate_diff
 
@@ -924,13 +924,13 @@ def track_diff(self, use, out, test, example, got):
         self._changesets[filename].append(info)
 
     def report_success(self, out, test, example, got):
-        if self._generate_diff is None:
-            return super().report_success(out, test, example, got)
+        if self._generate_diff:
+            return self.track_diff(True, out, test, example, got)
 
-        return self.track_diff(True, out, test, example, got)
+        return super().report_success(out, test, example, got)
 
     def report_failure(self, out, test, example, got):
-        if self._generate_diff is None:
-            return super().report_success(out, test, example, got)
+        if self._generate_diff:
+            self.track_diff(False, out, test, example, got)
 
-        return self.track_diff(False, out, test, example, got)
+        return super().report_failure(out, test, example, got)

From 58c321391e2a274a01e9b7ed723fef12da2c2c69 Mon Sep 17 00:00:00 2001
From: Sebastian Berg <sebastianb@nvidia.com>
Date: Fri, 6 Oct 2023 17:20:04 +0200
Subject: [PATCH 04/10] Fix overriding `report_failure` fully and fix
 styles/unused variables

---
 pytest_doctestplus/newhooks.py |  2 +-
 pytest_doctestplus/plugin.py   | 59 +++++++++++++++-------------------
 2 files changed, 27 insertions(+), 34 deletions(-)

diff --git a/pytest_doctestplus/newhooks.py b/pytest_doctestplus/newhooks.py
index 09a96c1..9d87105 100644
--- a/pytest_doctestplus/newhooks.py
+++ b/pytest_doctestplus/newhooks.py
@@ -2,4 +2,4 @@
 
 
 def pytest_doctestplus_diffhook(info):
-    """ called when a diff would be generated normally. """
\ No newline at end of file
+    """ called when a diff would be generated normally. """
diff --git a/pytest_doctestplus/plugin.py b/pytest_doctestplus/plugin.py
index ffb53e5..fda67ed 100644
--- a/pytest_doctestplus/plugin.py
+++ b/pytest_doctestplus/plugin.py
@@ -124,16 +124,18 @@ def pytest_addoption(parser):
                      help="Test only doctests. Implies usage of doctest-plus.")
 
     parser.addoption("--doctest-plus-generate-diff",
-                  help="Generate a diff for where expected output and real output "
-                  "differ.  The diff is printed to stdout if not using "
-                  "`--doctest-plus-generate-diff=overwrite` which causes editing of "
-                  "the original files.\n"
-                  "NOTE: Unless an in-pace build is picked "
-                  "up, python file paths may point to unexpected places. "
-                  "If inplace is not used, will create a temporary folder and "
-                  "use `git diff -p` to generate a diff.",
-                  choices=["diff", "overwrite"],
-                  action="store", nargs="?", default=False, const="diff")
+                     help=(
+                         "Generate a diff for where expected output and real "
+                         "output differ.  "
+                         "The diff is printed to stdout if not using "
+                         "`--doctest-plus-generate-diff=overwrite` which "
+                         "causes editing of the original files.\n"
+                         "NOTE: Unless an in-pace build is picked up, python "
+                         "file paths may point to unexpected places. "
+                         "If inplace is not used, will create a temporary "
+                         "folder and use `git diff -p` to generate a diff."),
+                     choices=["diff", "overwrite"],
+                     action="store", nargs="?", default=False, const="diff")
 
     parser.addini("text_file_format",
                   "Default format for docs. "
@@ -178,7 +180,7 @@ def pytest_addoption(parser):
 
 def pytest_addhooks(pluginmanager):
     from pytest_doctestplus import newhooks
-    method = pluginmanager.add_hookspecs(newhooks)
+    pluginmanager.add_hookspecs(newhooks)
 
 
 def get_optionflags(parent):
@@ -767,7 +769,7 @@ def write_modified_file(fname, new_fname, changes):
     # Sort in reversed order to edit the lines:
     bad_tests = []
     changes.sort(key=lambda x: (x["test_lineno"], x["example_lineno"]),
-                    reverse=True)
+                 reverse=True)
 
     with open(fname, "r") as f:
         text = f.readlines()
@@ -878,14 +880,24 @@ class DebugRunnerPlus(doctest.DebugRunner):
     _changesets = defaultdict(lambda: [])
     _generate_diff = False
 
-    def __init__(self, checker=None, verbose=None, optionflags=0, continue_on_failure=True, generate_diff=False):
+    def __init__(self, checker=None, verbose=None, optionflags=0,
+                 continue_on_failure=True, generate_diff=False):
         # generated_diff is False, "diff", or "inplace" (only need truthiness)
         DebugRunnerPlus._generate_diff = generate_diff
 
         super().__init__(checker=checker, verbose=verbose, optionflags=optionflags)
         self.continue_on_failure = continue_on_failure
 
+    def report_success(self, out, test, example, got):
+        if self._generate_diff:
+            return self.track_diff(True, out, test, example, got)
+
+        return super().report_success(out, test, example, got)
+
     def report_failure(self, out, test, example, got):
+        if self._generate_diff:
+            self.track_diff(False, out, test, example, got)
+
         failure = doctest.DocTestFailure(test, example, got)
         if self.continue_on_failure:
             out.append(failure)
@@ -913,24 +925,5 @@ def track_diff(self, use, out, test, example, got):
         doctestplus_diffhook(info=info)
         if not info["use"]:
             return
-        name = info["name"]
-        filename = info["filename"]
-        source = info["source"]
-        test_lineno = info["test_lineno"]
-        example_lineno = info["example_lineno"]
-        want = info["want"]
-        got = info["got"]
-
-        self._changesets[filename].append(info)
-
-    def report_success(self, out, test, example, got):
-        if self._generate_diff:
-            return self.track_diff(True, out, test, example, got)
-
-        return super().report_success(out, test, example, got)
-
-    def report_failure(self, out, test, example, got):
-        if self._generate_diff:
-            self.track_diff(False, out, test, example, got)
 
-        return super().report_failure(out, test, example, got)
+        self._changesets[info["filename"]].append(info)

From c7be6c59d41e73e4b21be6194e0f44a28009bd7c Mon Sep 17 00:00:00 2001
From: Sebastian Berg <sebastian@sipsolutions.net>
Date: Fri, 6 Oct 2023 22:53:46 +0200
Subject: [PATCH 05/10] Don't report fail/success and auto-enable doctest-only

---
 pytest_doctestplus/plugin.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/pytest_doctestplus/plugin.py b/pytest_doctestplus/plugin.py
index fda67ed..95ed4f2 100644
--- a/pytest_doctestplus/plugin.py
+++ b/pytest_doctestplus/plugin.py
@@ -208,6 +208,8 @@ def _is_numpy_ufunc(method):
 def pytest_configure(config):
     doctest_plugin = config.pluginmanager.getplugin('doctest')
     run_regular_doctest = config.option.doctestmodules and not config.option.doctest_plus
+    if config.option.doctest_plus_generate_diff:
+        config.option.doctest_only = True
     use_doctest_plus = config.getini(
         'doctest_plus') or config.option.doctest_plus or config.option.doctest_only
     use_doctest_ufunc = config.getini(
@@ -890,13 +892,15 @@ def __init__(self, checker=None, verbose=None, optionflags=0,
 
     def report_success(self, out, test, example, got):
         if self._generate_diff:
-            return self.track_diff(True, out, test, example, got)
+            self.track_diff(False, out, test, example, got)
+            return
 
         return super().report_success(out, test, example, got)
 
     def report_failure(self, out, test, example, got):
         if self._generate_diff:
-            self.track_diff(False, out, test, example, got)
+            self.track_diff(True, out, test, example, got)
+            return
 
         failure = doctest.DocTestFailure(test, example, got)
         if self.continue_on_failure:

From 3c0192b8551a297f75fe955ce281c08d969f2d4e Mon Sep 17 00:00:00 2001
From: Sebastian Berg <sebastian@sipsolutions.net>
Date: Fri, 6 Oct 2023 23:01:46 +0200
Subject: [PATCH 06/10] TST: Add a basic test and reset when reporting

---
 pytest_doctestplus/plugin.py |  2 ++
 tests/test_doctestplus.py    | 36 ++++++++++++++++++++++++++++++++++++
 2 files changed, 38 insertions(+)

diff --git a/pytest_doctestplus/plugin.py b/pytest_doctestplus/plugin.py
index 95ed4f2..7107afb 100644
--- a/pytest_doctestplus/plugin.py
+++ b/pytest_doctestplus/plugin.py
@@ -800,6 +800,8 @@ def write_modified_file(fname, new_fname, changes):
 def pytest_terminal_summary(terminalreporter, exitstatus, config):
     changesets = DebugRunnerPlus._changesets
     diff_mode = DebugRunnerPlus._generate_diff
+    DebugRunnerPlus._changesets = defaultdict(lambda: [])
+    DebugRunnerPlus._generate_diff = None
     all_bad_tests = []
     if not diff_mode:
         return  # we do not report or apply diffs
diff --git a/tests/test_doctestplus.py b/tests/test_doctestplus.py
index 4d2fc5c..b657436 100644
--- a/tests/test_doctestplus.py
+++ b/tests/test_doctestplus.py
@@ -1348,3 +1348,39 @@ def f():
     """, "utf-8")
     reprec = testdir.inline_run(str(testdir), "--doctest-plus")
     reprec.assertoutcome(failed=0, passed=0)
+
+
+def test_generate_diff_basic(testdir, capsys):
+    p = testdir.makepyfile("""
+        def f():
+            '''
+            >>> print(2)
+            4
+            >>> print(3)
+            5
+            '''
+            pass
+        """)
+    with open(p) as f:
+        original = f.read()
+
+    testdir.inline_run(p, "--doctest-plus-generate-diff")
+    diff = dedent("""
+         >>> print(2)
+    -    4
+    +    2
+         >>> print(3)
+    -    5
+    +    3
+    """)
+    captured = capsys.readouterr()
+    assert diff in captured.out
+
+    testdir.inline_run(p, "--doctest-plus-generate-diff=overwrite")
+    captured = capsys.readouterr()
+    assert "Applied fix to the following files" in captured.out
+
+    with open(p) as f:
+        result = f.read()
+
+    assert result == original.replace("4", "2").replace("5", "3")

From 2a56bf91908ae21f04bd7df025942aaa6265a524 Mon Sep 17 00:00:00 2001
From: Sebastian Berg <sebastian@sipsolutions.net>
Date: Fri, 6 Oct 2023 23:05:27 +0200
Subject: [PATCH 07/10] Fix formatting of README (its rst, not md...)

---
 README.rst | 56 ++++++++++++++++++++++++++++--------------------------
 1 file changed, 29 insertions(+), 27 deletions(-)

diff --git a/README.rst b/README.rst
index b0faf60..e3697b3 100644
--- a/README.rst
+++ b/README.rst
@@ -381,42 +381,44 @@ running the doctests with sphinx is not supported. To do this, add
 Fixing Existing Docstrings
 --------------------------
 The plugin has basic support to fix docstrings, this can be enabled by
-running `pytest` with `--doctest-plus-generate-diff`.
+running ``pytest`` with ``--doctest-plus-generate-diff``.
 Without further options, this will print out a diff and a list of files that
-would be modified.  Using `--doctest-plus-generate-diff=overwrite` will
+would be modified.  Using ``--doctest-plus-generate-diff=overwrite`` will
 modify the files in-place, so it is recommended to run the check first to
 verify the paths.
-You may wish to use e.g. `git commit -p` to review changes manually.
+You may wish to use e.g. ``git commit -p`` to review changes manually.
 
 The current diff generation is not very smart, so it does not account for
-existing `...`.  By default a diff is only generated for *failing* doctests.
+existing ``...``.  By default a diff is only generated for *failing* doctests.
 
 In general, a mass edit may wish to focus on a specific change and
 possibly include passing tests.  So you can hook into the behavior by
-adding a hook to your `conftest.py`:
-```python
-@pytest.hookimpl
-def pytest_doctestplus_diffhook(info):
-    info["use"] = True  # Overwrite all results (even successes)
-    if info["fileno"] is None:
-        # E.g. NumPy has C docstrings that cannot be found, we can add custom
-        # logic here to try and find these:
-        info["filename"] = ...
-        info["lineno"] = ...
-```
-Where `info` is a dictionary containing the following items:
-* `use`: `True` or `False` signalling whether to apply the diff.  This is
-  set to `False` if a doctest succeeded and `True` if the doctest failed.
-* `name`: The name of the test (e.g. the function being documented)
-* `filename`: The file that contains the test (this can be wrong in certain
-  situation and in that case `test_lineno` will be wrong as well).
-* `source`: The source code that was executed for this test
-* `test_lineno`: The line of code where the example block (or function) starts.
-  In some cases, the test file cannot be found and the lineno will be `None`,
+adding a hook to your ``conftest.py``::
+
+    @pytest.hookimpl
+    def pytest_doctestplus_diffhook(info):
+        info["use"] = True  # Overwrite all results (even successes)
+        if info["fileno"] is None:
+            # E.g. NumPy has C docstrings that cannot be found, we can add
+            # custom logic here to try and find these:
+            info["filename"] = ...
+            info["lineno"] = ...
+
+Where ``info`` is a dictionary containing the following items:
+
+* ``use``: ``True`` or ``False`` signalling whether to apply the diff.  This is
+  set to ``False`` if a doctest succeeded and ``True`` if the doctest failed.
+* ``name``: The name of the test (e.g. the function being documented)
+* ``filename``: The file that contains the test (this can be wrong in certain
+  situation and in that case ``test_lineno`` will be wrong as well).
+* ``source``: The source code that was executed for this test
+* ``test_lineno``: The line of code where the example block (or function) starts.
+  In some cases, the test file cannot be found and the lineno will be ``None``,
   you can manually try to fix these.
-* `example_lineno`: The line number of the example snippet (individual `>>>`).
-* `want`: The current documentation.
-* `got`: The result of executing the example.
+* ``example_lineno``: The line number of the example snippet
+  (individual ``>>>``).
+* ``want``: The current documentation.
+* ``got``: The result of executing the example.
 
 You can modify the dictionary in-place to modify the behavior.
 

From d74ff2972c5d6ce178227f2afc99daa4b23f7b43 Mon Sep 17 00:00:00 2001
From: Sebastian Berg <sebastian@sipsolutions.net>
Date: Sat, 9 Dec 2023 14:19:54 +0100
Subject: [PATCH 08/10] Apply suggestions from code review
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Brigitta Sipőcz <b.sipocz@gmail.com>
---
 README.rst                   | 10 +++++-----
 pytest_doctestplus/plugin.py |  6 +++---
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/README.rst b/README.rst
index e3697b3..442c11c 100644
--- a/README.rst
+++ b/README.rst
@@ -41,8 +41,8 @@ providing the following features:
 * optional inclusion of ``*.rst`` files for doctests (see `Setup and Configuration`_)
 * optional inclusion of doctests in docstrings of Numpy ufuncs
 
-Further, `pytest-doctestplus` supports editing files to fix incorrect docstrings
-(See `Fixing Existing Docstrings`).
+Further, ``pytest-doctestplus`` supports editing files to fix incorrect docstrings
+(See `Fixing Existing Docstrings`_).
 
 .. _pytest-remotedata: https://github.com/astropy/pytest-remotedata
 
@@ -386,13 +386,13 @@ Without further options, this will print out a diff and a list of files that
 would be modified.  Using ``--doctest-plus-generate-diff=overwrite`` will
 modify the files in-place, so it is recommended to run the check first to
 verify the paths.
-You may wish to use e.g. ``git commit -p`` to review changes manually.
+You may wish to review changes manually and only commit some patches e.g. using ``git commit --patch``.
 
-The current diff generation is not very smart, so it does not account for
+The current diff generation is still very basic, for example, it does not account for
 existing ``...``.  By default a diff is only generated for *failing* doctests.
 
 In general, a mass edit may wish to focus on a specific change and
-possibly include passing tests.  So you can hook into the behavior by
+possibly include passing tests.  So you can opt-in into the behavior by
 adding a hook to your ``conftest.py``::
 
     @pytest.hookimpl
diff --git a/pytest_doctestplus/plugin.py b/pytest_doctestplus/plugin.py
index 7107afb..bbfffbb 100644
--- a/pytest_doctestplus/plugin.py
+++ b/pytest_doctestplus/plugin.py
@@ -125,14 +125,14 @@ def pytest_addoption(parser):
 
     parser.addoption("--doctest-plus-generate-diff",
                      help=(
-                         "Generate a diff for where expected output and real "
+                         "Generate a diff where expected output and actual "
                          "output differ.  "
                          "The diff is printed to stdout if not using "
                          "`--doctest-plus-generate-diff=overwrite` which "
                          "causes editing of the original files.\n"
                          "NOTE: Unless an in-pace build is picked up, python "
                          "file paths may point to unexpected places. "
-                         "If inplace is not used, will create a temporary "
+                         "If `"overwrite"` is not used, will create a temporary "
                          "folder and use `git diff -p` to generate a diff."),
                      choices=["diff", "overwrite"],
                      action="store", nargs="?", default=False, const="diff")
@@ -886,7 +886,7 @@ class DebugRunnerPlus(doctest.DebugRunner):
 
     def __init__(self, checker=None, verbose=None, optionflags=0,
                  continue_on_failure=True, generate_diff=False):
-        # generated_diff is False, "diff", or "inplace" (only need truthiness)
+        # generated_diff is False, "diff", or "overwrite" (only need truthiness)
         DebugRunnerPlus._generate_diff = generate_diff
 
         super().__init__(checker=checker, verbose=verbose, optionflags=optionflags)

From f43c759ef4a0f9383f612ae9882cf52b848ca90b Mon Sep 17 00:00:00 2001
From: Sebastian Berg <sebastian@sipsolutions.net>
Date: Sat, 9 Dec 2023 14:22:56 +0100
Subject: [PATCH 09/10] Update pytest_doctestplus/plugin.py

---
 pytest_doctestplus/plugin.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pytest_doctestplus/plugin.py b/pytest_doctestplus/plugin.py
index bbfffbb..64a8401 100644
--- a/pytest_doctestplus/plugin.py
+++ b/pytest_doctestplus/plugin.py
@@ -132,7 +132,7 @@ def pytest_addoption(parser):
                          "causes editing of the original files.\n"
                          "NOTE: Unless an in-pace build is picked up, python "
                          "file paths may point to unexpected places. "
-                         "If `"overwrite"` is not used, will create a temporary "
+                         "If 'overwrite' is not used, will create a temporary "
                          "folder and use `git diff -p` to generate a diff."),
                      choices=["diff", "overwrite"],
                      action="store", nargs="?", default=False, const="diff")

From 269873e231c72ba097a92166606471251c304089 Mon Sep 17 00:00:00 2001
From: Sebastian Berg <sebastian@sipsolutions.net>
Date: Sat, 9 Dec 2023 15:11:06 +0100
Subject: [PATCH 10/10] DOC: Add small note in CHANGES.rst

---
 CHANGES.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/CHANGES.rst b/CHANGES.rst
index 2cab2eb..7e39f2a 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -9,6 +9,8 @@
 
 - Respect pytest ``--import-mode``. [#233]
 
+- Ability to update documentation based on actual output. [#227]
+
 
 1.0.0 (2023-08-11)
 ==================