Merge branch 'pip-tools-integration'

* pip-tools-integration:
  Revert "Exclude packages required only by unsafe packages"
  Exclude unsafe packages' dependencies when `--allow-unsafe` is not in use

prequ/resolver.py

@@ -15,7 +15,8 @@
 from .exceptions import UnsupportedConstraint
 from .logging import log
 from .utils import (format_requirement, format_specifier, full_groupby,
-                    is_pinned_requirement, key_from_req, is_vcs_link)
+                    is_pinned_requirement, key_from_req, is_vcs_link,
+                    UNSAFE_PACKAGES)
 
 green = partial(click.style, fg='green')
 magenta = partial(click.style, fg='magenta')
@@ -49,7 +50,7 @@ def __str__(self):
 
 
 class Resolver(object):
-    def __init__(self, constraints, repository, cache=None, prereleases=False, clear_caches=False):
+    def __init__(self, constraints, repository, cache=None, prereleases=False, clear_caches=False, allow_unsafe=False):
         """
         This class resolves a given set of constraints (a collection of
         InstallRequirement objects) by consulting the given Repository and the
@@ -63,6 +64,7 @@ def __init__(self, constraints, repository, cache=None, prereleases=False, clear
         self.dependency_cache = cache
         self.prereleases = prereleases
         self.clear_caches = clear_caches
+        self.allow_unsafe = allow_unsafe
 
     @property
     def constraints(self):
@@ -188,10 +190,14 @@ def _resolve_one_round(self):
         # Find the new set of secondary dependencies
         log.debug('')
         log.debug('Finding secondary dependencies:')
+
+        ungrouped = []
+        for best_match in best_matches:
+            for dep in self._iter_dependencies(best_match):
+                if self.allow_unsafe or dep.name not in UNSAFE_PACKAGES:
+                    ungrouped.append(dep)
         # Grouping constraints to make clean diff between rounds
-        theirs = set(self._group_constraints(dep
-                     for best_match in best_matches
-                     for dep in self._iter_dependencies(best_match)))
+        theirs = set(self._group_constraints(ungrouped))
 
         # NOTE: We need to compare RequirementSummary objects, since
         # InstallRequirement does not define equality

prequ/scripts/compile_in.py

@@ -155,7 +155,7 @@ def cli(verbose, silent, dry_run, pre, rebuild, find_links, index_url,
 
     try:
         resolver = Resolver(constraints, repository, prereleases=pre,
-                            clear_caches=rebuild)
+                            clear_caches=rebuild, allow_unsafe=allow_unsafe)
         results = resolver.resolve()
         if generate_hashes:
             hashes = resolver.resolve_hashes(results)
@@ -202,9 +202,8 @@ def cli(verbose, silent, dry_run, pre, rebuild, find_links, index_url,
                           default_index_url=repository.DEFAULT_INDEX_URL,
                           index_urls=repository.finder.index_urls,
                           trusted_hosts=pip_options.trusted_hosts,
-                          format_control=repository.finder.format_control,
                           find_links=repository.finder.find_links,
-                          allow_unsafe=allow_unsafe,
+                          format_control=repository.finder.format_control,
                           silent=silent)
     writer.write(results=results,
                  reverse_dependencies=reverse_dependencies,

prequ/utils.py

@@ -22,6 +22,8 @@ def safeint(s):
 
 pip_version_info = tuple(safeint(digit) for digit in pip.__version__.split('.'))
 
+UNSAFE_PACKAGES = {'setuptools', 'distribute', 'pip'}
+
 
 def assert_compatible_pip_version():
     # Make sure we're using a reasonably modern version of pip
@@ -44,16 +46,6 @@ def key_from_req(req):
     return key
 
 
-def name_from_req(req):
-    """Get the name of the requirement"""
-    if hasattr(req, 'project_name'):
-        # pip 8.1.1 or below, using pkg_resources
-        return req.project_name
-    else:
-        # pip 8.1.2 or above, using packaging
-        return req.name
-
-
 def comment(text):
     return style(text, fg='green')
 
@@ -68,17 +60,15 @@ def make_install_requirement(name, version, extras, constraint=False):
     return InstallRequirement.from_line('{}{}=={}'.format(name, extras_string, str(version)), constraint=constraint)
 
 
-def format_requirement(ireq, include_specifier=True):
+def format_requirement(ireq):
     """
     Generic formatter for pretty printing InstallRequirements to the terminal
     in a less verbose way than using its `__str__` method.
     """
     if ireq.editable or is_vcs_link(ireq):
         line = '{}{}'.format('-e ' if ireq.editable else '', ireq.link)
-    elif include_specifier:
-        line = str(ireq.req).lower()
     else:
-        line = name_from_req(ireq.req).lower()
+        line = str(ireq.req).lower()
     return line
 
 

prequ/writer.py

@@ -5,14 +5,14 @@
 from .click import unstyle
 from .fileutils import AtomicSaver
 from .logging import log
-from .utils import comment, format_requirement, dedup
+from .utils import comment, format_requirement, dedup, UNSAFE_PACKAGES
 
 
 class OutputWriter(object):
     def __init__(self, src_files, dst_file, dry_run, emit_header, emit_index,
                  emit_trusted_host, annotate, generate_hashes,
                  default_index_url, index_urls, trusted_hosts, find_links,
-                 format_control, allow_unsafe=False, silent=False):
+                 format_control, silent=False):
         self.src_files = src_files
         self.dst_file = dst_file
         self.dry_run = dry_run
@@ -26,7 +26,6 @@ def __init__(self, src_files, dst_file, dry_run, emit_header, emit_index,
         self.trusted_hosts = trusted_hosts
         self.find_links = find_links
         self.format_control = format_control
-        self.allow_unsafe = allow_unsafe
         self.silent = silent
 
     def _sort_key(self, ireq):
@@ -87,24 +86,11 @@ def _iter_lines(self, results, reverse_dependencies, primary_packages, hashes):
         for line in self.write_flags():
             yield line
 
-        UNSAFE_PACKAGES = {'setuptools', 'distribute', 'pip'}
-
-        packages = set()
-        unsafe_packages = set()
-        ignored_packages = set()
-        for r in results:
-            if r.name in UNSAFE_PACKAGES:
-                unsafe_packages.add(r)
-            else:
-                reverse = reverse_dependencies.get(r.name.lower())
-                if reverse and all(name in UNSAFE_PACKAGES for name in reverse):
-                    ignored_packages.add(r)
-                else:
-                    packages.add(r)
+        unsafe_packages = {r for r in results if r.name in UNSAFE_PACKAGES}
+        packages = {r for r in results if r.name not in UNSAFE_PACKAGES}
 
         packages = sorted(packages, key=self._sort_key)
         unsafe_packages = sorted(unsafe_packages, key=self._sort_key)
-        ignored_packages = sorted(ignored_packages, key=self._sort_key)
 
         for ireq in packages:
             line = self._format_requirement(ireq, reverse_dependencies, primary_packages, hashes=hashes)
@@ -115,30 +101,7 @@ def _iter_lines(self, results, reverse_dependencies, primary_packages, hashes):
             yield comment('# The following packages are considered to be unsafe in a requirements file:')
 
             for ireq in unsafe_packages:
-                line = self._format_requirement(
-                    ireq, reverse_dependencies, primary_packages,
-                    hashes=hashes if self.allow_unsafe else None,
-                    include_specifier=self.allow_unsafe)
-                if self.allow_unsafe:
-                    yield line
-                else:
-                    yield comment('# ' + line)
-
-        if ignored_packages:
-            yield ''
-            yield comment(
-                '# The following packages are required only by packages'
-                ' considered to be unsafe in a requirements file:')
-
-            for ireq in ignored_packages:
-                line = self._format_requirement(
-                    ireq, reverse_dependencies, primary_packages,
-                    hashes=hashes if self.allow_unsafe else None,
-                    include_specifier=self.allow_unsafe)
-                if self.allow_unsafe:
-                    yield line
-                else:
-                    yield comment('# ' + line)
+                yield self._format_requirement(ireq, reverse_dependencies, primary_packages, hashes=hashes)
 
     def write(self, results, reverse_dependencies, primary_packages, hashes):
         with ExitStack() as stack:
@@ -153,8 +116,8 @@ def write(self, results, reverse_dependencies, primary_packages, hashes):
                     f.write(unstyle(line).encode('utf-8'))
                     f.write(os.linesep.encode('utf-8'))
 
-    def _format_requirement(self, ireq, reverse_dependencies, primary_packages, include_specifier=True, hashes=None):
-        line = format_requirement(ireq, include_specifier=include_specifier)
+    def _format_requirement(self, ireq, reverse_dependencies, primary_packages, hashes=None):
+        line = format_requirement(ireq)
 
         ireq_hashes = (hashes if hashes is not None else {}).get(ireq)
         if ireq_hashes:

tests/fixtures/fake-index.json

@@ -66,6 +66,11 @@
   "gnureadline": {
     "6.3.3": {"": []}
   },
+  "html5lib": {
+    "0.999999999": {"": [
+      "setuptools>=18.5"
+    ]}
+  },
   "ipython": {
     "2.1.0": {
       "": ["gnureadline"],

tests/test_minimal_upgrade.py

@@ -1,6 +1,15 @@
 import pytest
 from prequ.repositories import LocalRequirementsRepository
-from prequ.utils import name_from_req
+
+
+def name_from_req(req):
+    """Get the name of the requirement"""
+    if hasattr(req, 'project_name'):
+        # pip 8.1.1 or below, using pkg_resources
+        return req.project_name
+    else:
+        # pip 8.1.2 or above, using packaging
+        return req.name
 
 
 @pytest.mark.parametrize(

tests/test_resolver.py

@@ -101,7 +101,10 @@
         (['Flask', ('click', True), ('itsdangerous', True)],
          ['flask==0.10.1', 'itsdangerous==0.24', 'markupsafe==0.23',
           'jinja2==2.7.3', 'werkzeug==0.10.4']
-         )
+         ),
+
+        # Exclude package dependcy of setuptools as it is unsafe.
+        (['html5lib'], ['html5lib==0.999999999']),
     ])
 )
 def test_resolver(resolver, from_line, input, expected, prereleases):

tests/test_utils.py

@@ -7,7 +7,6 @@
 def test_format_requirement(from_line):
     ireq = from_line('test==1.2')
     assert format_requirement(ireq) == 'test==1.2'
-    assert format_requirement(ireq, include_specifier=False) == 'test'
 
 
 def test_format_requirement_editable(from_editable):

tests/test_writer.py

@@ -58,46 +58,3 @@ def test_format_requirement_not_for_primary(from_line, writer):
                                        reverse_dependencies,
                                        primary_packages=['test']) ==
             'test==1.2')
-
-
-def test_iter_lines_ignores_dependencies_of_unsafe(from_line, writer):
-    lines = writer._iter_lines(
-        results=[from_line('test'), from_line('setuptools'), from_line('appdirs')],
-        reverse_dependencies={'appdirs': ['setuptools'], 'setuptools': ['test']},
-        primary_packages=['test'],
-        hashes=None)
-    expected = [
-        '\x1b[32m# This file is autogenerated by Prequ.  To update, run:\x1b[0m',
-        '\x1b[32m#\x1b[0m',
-        '\x1b[32m#   prequ update\x1b[0m',
-        '\x1b[32m#\x1b[0m',
-        'test',
-        '',
-        '\x1b[32m# The following packages are considered to be unsafe in a requirements file:\x1b[0m',
-        '\x1b[32m# setuptools                \x1b[32m# via test\x1b[0m\x1b[0m',
-        '',
-        ('\x1b[32m# The following packages are required only by packages'
-         ' considered to be unsafe in a requirements file:\x1b[0m'),
-        '\x1b[32m# appdirs                   \x1b[32m# via setuptools\x1b[0m\x1b[0m',
-    ]
-    assert list(lines) == expected
-
-
-def test_iter_lines_does_not_ignore_dependencies_of_unsafe_and_package(from_line, writer):
-    lines = writer._iter_lines(
-        results=[from_line('test'), from_line('setuptools'), from_line('appdirs')],
-        reverse_dependencies={'appdirs': ['test', 'setuptools'], 'setuptools': ['test']},
-        primary_packages=['test'],
-        hashes=None)
-    expected = [
-        '\x1b[32m# This file is autogenerated by Prequ.  To update, run:\x1b[0m',
-        '\x1b[32m#\x1b[0m',
-        '\x1b[32m#   prequ update\x1b[0m',
-        '\x1b[32m#\x1b[0m',
-        'appdirs                   \x1b[32m# via setuptools, test\x1b[0m',
-        'test',
-        '',
-        '\x1b[32m# The following packages are considered to be unsafe in a requirements file:\x1b[0m',
-        '\x1b[32m# setuptools                \x1b[32m# via test\x1b[0m\x1b[0m',
-    ]
-    assert list(lines) == expected