Improve pipenv update and add pipenv upgrade command (#5617)

* Split apart core using pycharm refactor move methods.

* move init to remove cicular import.

* Fix imports.

* Check in concept for pipenv upgrade command

* Fix upgrade command expectation on how it updates the lockfile.

* Actually write the result to the Pipfile, and fix secondary bug with items not being written to the Pipfile.

* Fix issue where package being upgraded already exists.

* Add news fragment.

* Integrate upgrade with a refactor of update.

* Handle cases where there is nothing to upgrade.

* Add lock-only option.

news/5617.feature.rst

@@ -0,0 +1,4 @@
+Provide a more powerful solution than ``--keep-outdated`` and ``--selective-upgrade`` which are deprecated for removal.
+Introducing the ``pipenv upgrade`` command which takes the same package specifiers as ``pipenv install`` and
+updates the ``Pipfile`` and ``Pipfile.lock`` with a valid lock resolution that only effects the specified packages and their dependencies.
+Additionally, the ``pipenv update`` command has been updated to use the ``pipenv upgrade`` routine when packages are provided, which will install sync the new lock file as well.

pipenv/cli/command.py

@@ -20,6 +20,7 @@
     sync_options,
     system_option,
     uninstall_options,
+    upgrade_options,
     verbose_option,
 )
 from pipenv.utils.dependencies import get_lockfile_section_using_pipfile_category
@@ -253,6 +254,39 @@ def install(state, **kwargs):
     )
 
 
+@cli.command(
+    short_help="Resolves provided packages and adds them to Pipfile, or (if no packages are given), merges results to Pipfile.lock",
+    context_settings=subcommand_context,
+)
+@system_option
+@site_packages_option
+@install_options
+@upgrade_options
+@pass_state
+def upgrade(state, **kwargs):
+    from pipenv.routines.update import upgrade
+    from pipenv.utils.project import ensure_project
+
+    ensure_project(
+        state.project,
+        python=state.python,
+        pypi_mirror=state.pypi_mirror,
+        warn=(not state.quiet),
+        site_packages=state.site_packages,
+        clear=state.clear,
+    )
+
+    upgrade(
+        state.project,
+        pre=state.installstate.pre,
+        packages=state.installstate.packages,
+        editable_packages=state.installstate.editables,
+        categories=state.installstate.categories,
+        system=state.system,
+        lock_only=state.installstate.lock_only,
+    )
+
+
 @cli.command(
     short_help="Uninstalls a provided package and removes it from Pipfile.",
     context_settings=subcommand_context,
@@ -332,7 +366,6 @@ def lock(ctx, state, **kwargs):
     pre = state.installstate.pre
     do_lock(
         state.project,
-        ctx=ctx,
         clear=state.clear,
         pre=pre,
         keep_outdated=state.installstate.keep_outdated,
@@ -534,76 +567,32 @@ def check(
 @option("--outdated", is_flag=True, default=False, help="List out-of-date dependencies.")
 @option("--dry-run", is_flag=True, default=None, help="List out-of-date dependencies.")
 @install_options
+@upgrade_options
 @pass_state
 @pass_context
 def update(ctx, state, bare=False, dry_run=None, outdated=False, **kwargs):
-    """Runs lock, then sync."""
-    from pipenv.routines.install import do_sync
-    from pipenv.routines.lock import do_lock
-    from pipenv.routines.outdated import do_outdated
-    from pipenv.utils.project import ensure_project
+    """Runs lock when no packages are specified, or upgrade, and then sync."""
+    from pipenv.routines.update import do_update
 
-    ensure_project(
+    do_update(
         state.project,
         python=state.python,
-        pypi_mirror=state.pypi_mirror,
-        warn=(not state.quiet),
         site_packages=state.site_packages,
         clear=state.clear,
-    )
-    if not outdated:
-        outdated = bool(dry_run)
-    if outdated:
-        do_outdated(
-            state.project,
-            clear=state.clear,
-            pre=state.installstate.pre,
-            pypi_mirror=state.pypi_mirror,
-        )
-    packages = [p for p in state.installstate.packages if p]
-    editable = [p for p in state.installstate.editables if p]
-    if not packages:
-        echo(
-            "{} {} {} {}{}".format(
-                style("Running", bold=True),
-                style("$ pipenv lock", fg="yellow", bold=True),
-                style("then", bold=True),
-                style("$ pipenv sync", fg="yellow", bold=True),
-                style(".", bold=True),
-            )
-        )
-    else:
-        for package in packages + editable:
-            if package not in state.project.all_packages:
-                echo(
-                    "{}: {} was not found in your Pipfile! Aborting."
-                    "".format(
-                        style("Warning", fg="red", bold=True),
-                        style(package, fg="green", bold=True),
-                    ),
-                    err=True,
-                )
-                ctx.abort()
-    do_lock(
-        state.project,
-        ctx=ctx,
-        clear=state.clear,
         pre=state.installstate.pre,
-        keep_outdated=state.installstate.keep_outdated,
         pypi_mirror=state.pypi_mirror,
-        write=not state.quiet,
-    )
-    do_sync(
-        state.project,
+        keep_outdated=state.installstate.keep_outdated,
+        system=False,
+        packages=state.installstate.packages,
+        editable_packages=state.installstate.editables,
         dev=state.installstate.dev,
-        python=state.python,
         bare=bare,
-        dont_upgrade=not state.installstate.keep_outdated,
-        user=False,
-        clear=state.clear,
-        unused=False,
-        pypi_mirror=state.pypi_mirror,
         extra_pip_args=state.installstate.extra_pip_args,
+        categories=state.installstate.categories,
+        quiet=state.quiet,
+        dry_run=dry_run,
+        outdated=outdated,
+        lock_only=state.installstate.lock_only,
     )
 
 

pipenv/cli/options.py

@@ -159,7 +159,7 @@ def callback(ctx, param, value):
             click.secho(
                 "The flag --keep-outdated has been deprecated for removal.  "
                 "The flag does not respect package resolver results and leads to inconsistent lock files.  "
-                "Please pin relevant requirements in your Pipfile and discontinue use of this flag.",
+                "Consider using the new `pipenv upgrade` command to selectively upgrade packages.",
                 fg="yellow",
                 bold=True,
                 err=True,
@@ -182,6 +182,15 @@ def selective_upgrade_option(f):
     def callback(ctx, param, value):
         state = ctx.ensure_object(State)
         state.installstate.selective_upgrade = value
+        if value:
+            click.secho(
+                "The flag --selective-upgrade has been deprecated for removal.  "
+                "The flag is buggy and leads to inconsistent lock files.  "
+                "Consider using the new `pipenv upgrade` command to selectively upgrade packages.",
+                fg="yellow",
+                bold=True,
+                err=True,
+            )
         return value
 
     return option(
@@ -503,6 +512,23 @@ def callback(ctx, param, value):
     )(f)
 
 
+def lock_only_option(f):
+    def callback(ctx, param, value):
+        state = ctx.ensure_object(State)
+        state.installstate.lock_only = value
+        return value
+
+    return option(
+        "--lock-only",
+        is_flag=True,
+        default=False,
+        help="Only update lock file (specifiers not added to Pipfile).",
+        callback=callback,
+        type=click_types.BOOL,
+        expose_value=False,
+    )(f)
+
+
 def setup_verbosity(ctx, param, value):
     if not value:
         return
@@ -586,6 +612,11 @@ def install_options(f):
     return f
 
 
+def upgrade_options(f):
+    f = lock_only_option(f)
+    return f
+
+
 def general_options(f):
     f = common_options(f)
     f = site_packages_option(f)

pipenv/project.py

@@ -26,7 +26,6 @@
 from pipenv.utils.dependencies import (
     get_canonical_names,
     is_editable,
-    is_star,
     pep423_name,
     python_version,
 )
@@ -977,12 +976,12 @@ def add_package_to_pipfile(self, package, dev=False, category=None):
         # Set empty group if it doesn't exist yet.
         if category not in p:
             p[category] = {}
-        name = self.get_package_name_in_pipfile(req_name, category=category)
-        if name and is_star(converted):
-            # Skip for wildcard version
-            return
         # Add the package to the group.
-        p[category][name or pep423_name(req_name)] = converted
+        name = self.get_package_name_in_pipfile(req_name, category=category)
+        normalized_name = pep423_name(req_name)
+        if name and name != normalized_name:
+            self.remove_package_from_pipfile(name, category=category)
+        p[category][normalized_name] = converted
         # Write Pipfile.
         self.write_toml(p)
 

pipenv/routines/lock.py

@@ -10,7 +10,6 @@
 
 def do_lock(
     project,
-    ctx=None,
     system=False,
     clear=False,
     pre=False,
@@ -27,7 +26,6 @@ def do_lock(
         if not project.lockfile_exists:
             raise exceptions.PipenvOptionsError(
                 "--keep-outdated",
-                ctx=ctx,
                 message="Pipfile.lock must exist to use --keep-outdated!",
             )
         cached_lockfile = project.lockfile_content