Fix --source performance regression. #1037

CHANGES.rst

@@ -24,7 +24,10 @@ want to know what's different in 5.0 since 4.5.x, see :ref:`whatsnew5x`.
 Unreleased
 ----------
 
-Nothing yet.
+- When using ``--source`` on a large source tree, v5.x was slower than previous
+  versions.  This performance regression is now fixed, closing `issue 1037`_.
+
+.. _issue 1037: https://github.com/nedbat/coveragepy/issues/1037
 
 
 .. _changes_53:

coverage/control.py

@@ -4,6 +4,7 @@
 """Core control stuff for coverage.py."""
 
 import atexit
+import collections
 import contextlib
 import os
 import os.path
@@ -737,9 +738,12 @@ def _post_save_work(self):
         # Touch all the files that could have executed, so that we can
         # mark completely unexecuted files as 0% covered.
         if self._data is not None:
+            file_paths = collections.defaultdict(list)
             for file_path, plugin_name in self._inorout.find_possibly_unexecuted_files():
                 file_path = self._file_mapper(file_path)
-                self._data.touch_file(file_path, plugin_name)
+                file_paths[plugin_name].append(file_path)
+            for plugin_name, paths in file_paths.items():
+                self._data.touch_files(paths, plugin_name)
 
         if self.config.note:
             self._warn("The '[run] note' setting is no longer supported.")

coverage/sqldata.py

@@ -167,7 +167,8 @@ class CoverageData(SimpleReprMixin):
     To record data for contexts, use :meth:`set_context` to set a context to
     be used for subsequent :meth:`add_lines` and :meth:`add_arcs` calls.
 
-    To add a source file without any measured data, use :meth:`touch_file`.
+    To add a source file without any measured data, use :meth:`touch_file`,
+    or :meth:`touch_files` for a list of such files.
 
     Write the data to its file with :meth:`write`.
 
@@ -536,16 +537,26 @@ def touch_file(self, filename, plugin_name=""):
         `plugin_name` is the name of the plugin responsible for this file. It is used
         to associate the right filereporter, etc.
         """
+        self.touch_files([filename], plugin_name)
+
+    def touch_files(self, filenames, plugin_name=""):
+        """Ensure that `filenames` appear in the data, empty if needed.
+
+        `plugin_name` is the name of the plugin responsible for these files. It is used
+        to associate the right filereporter, etc.
+        """
         if self._debug.should('dataop'):
-            self._debug.write("Touching %r" % (filename,))
+            self._debug.write("Touching %r" % (filenames,))
         self._start_using()
-        if not self._has_arcs and not self._has_lines:
-            raise CoverageException("Can't touch files in an empty CoverageData")
-
-        self._file_id(filename, add=True)
-        if plugin_name:
-            # Set the tracer for this file
-            self.add_file_tracers({filename: plugin_name})
+        with self._connect(): # Use this to get one transaction.
+            if not self._has_arcs and not self._has_lines:
+                raise CoverageException("Can't touch files in an empty CoverageData")
+
+            for filename in filenames:
+                self._file_id(filename, add=True)
+                if plugin_name:
+                    # Set the tracer for this file
+                    self.add_file_tracers({filename: plugin_name})
 
     def update(self, other_data, aliases=None):
         """Update this data with data from several other :class:`CoverageData` instances.