[Upgrade] Add az upgrade command

src/azure-cli-core/azure/cli/core/__init__.py

@@ -48,7 +48,8 @@ def __init__(self, **kwargs):
             register_ids_argument, register_global_subscription_argument)
         from azure.cli.core.cloud import get_active_cloud
         from azure.cli.core.commands.transform import register_global_transforms
-        from azure.cli.core._session import ACCOUNT, CONFIG, SESSION, INDEX
+        from azure.cli.core._session import ACCOUNT, CONFIG, SESSION, INDEX, VERSIONS
+        from azure.cli.core.util import handle_version_update
 
         from knack.util import ensure_dir
 
@@ -64,6 +65,8 @@ def __init__(self, **kwargs):
         CONFIG.load(os.path.join(azure_folder, 'az.json'))
         SESSION.load(os.path.join(azure_folder, 'az.sess'), max_age=3600)
         INDEX.load(os.path.join(azure_folder, 'commandIndex.json'))
+        VERSIONS.load(os.path.join(azure_folder, 'versionCheck.json'))
+        handle_version_update()
 
         self.cloud = get_active_cloud(self)
         logger.debug('Current cloud config:\n%s', str(self.cloud.name))

src/azure-cli-core/azure/cli/core/extension/_resolve.py

@@ -74,14 +74,14 @@ def resolve_from_index(extension_name, cur_version=None, index_url=None, target_
 
     candidates_sorted = sorted(candidates, key=lambda c: parse_version(c['metadata']['version']), reverse=True)
     logger.debug("Candidates %s", [c['filename'] for c in candidates_sorted])
-    logger.debug("Choosing the latest of the remaining candidates.")
 
     if target_version:
         try:
             chosen = [c for c in candidates_sorted if c['metadata']['version'] == target_version][0]
         except IndexError:
             raise NoExtensionCandidatesError('Extension with version {} not found'.format(target_version))
     else:
+        logger.debug("Choosing the latest of the remaining candidates.")
         chosen = candidates_sorted[0]
 
     logger.debug("Chosen %s", chosen)

src/azure-cli-core/azure/cli/core/extension/operations.py

@@ -298,7 +298,9 @@ def update_extension(cmd=None, extension_name=None, index_url=None, pip_extra_in
             download_url, ext_sha256 = resolve_from_index(extension_name, cur_version=cur_version, index_url=index_url)
         except NoExtensionCandidatesError as err:
             logger.debug(err)
-            raise CLIError("No updates available for '{}'. Use --debug for more information.".format(extension_name))
+            msg = "No updates available for '{}'. Use --debug for more information.".format(extension_name)
+            logger.warning(msg)
+            return
         # Copy current version of extension to tmp directory in case we need to restore it after a failed install.
         backup_dir = os.path.join(tempfile.mkdtemp(), extension_name)
         extension_path = ext.path
@@ -356,6 +358,45 @@ def list_available_extensions(index_url=None, show_details=False):
     return results
 
 
+def list_versions(extension_name, index_url=None):
+    index_data = get_index_extensions(index_url=index_url)
+
+    try:
+        exts = index_data[extension_name]
+    except Exception:
+        raise CLIError('Extension {} not found.'.format(extension_name))
+
+    try:
+        installed_ext = get_extension(extension_name, ext_type=WheelExtension)
+    except ExtensionNotInstalledException:
+        installed_ext = None
+
+    results = []
+    latest_compatible_version = None
+
+    for ext in sorted(exts, key=lambda c: parse_version(c['metadata']['version']), reverse=True):
+        compatible = ext_compat_with_cli(ext['metadata'])[0]
+        ext_version = ext['metadata']['version']
+        if latest_compatible_version is None and compatible:
+            latest_compatible_version = ext_version
+        installed = ext_version == installed_ext.version if installed_ext else False
+        if installed and parse_version(latest_compatible_version) > parse_version(installed_ext.version):
+            installed = str(True) + ' (upgrade available)'
+        version = ext['metadata']['version']
+        if latest_compatible_version == ext_version:
+            version = version + ' (max compatible version)'
+        results.append({
+            'name': extension_name,
+            'version': version,
+            'preview': ext['metadata'].get(EXT_METADATA_ISPREVIEW, False),
+            'experimental': ext['metadata'].get(EXT_METADATA_ISEXPERIMENTAL, False),
+            'installed': installed,
+            'compatible': compatible
+        })
+    results.reverse()
+    return results
+
+
 def reload_extension(extension_name, extension_module=None):
     return reload_module(extension_module if extension_module else get_extension_modname(ext_name=extension_name))
 

src/azure-cli-core/azure/cli/core/extension/tests/latest/test_extension_commands.py

@@ -304,13 +304,18 @@ def test_update_extension_not_found(self):
         self.assertEqual(str(err.exception), 'The extension {} is not installed.'.format(MY_EXT_NAME))
 
     def test_update_extension_no_updates(self):
+        logger_msgs = []
+
+        def mock_log_warning(_, msg):
+            logger_msgs.append(msg)
+
         add_extension(cmd=self.cmd, source=MY_EXT_SOURCE)
         ext = show_extension(MY_EXT_NAME)
         self.assertEqual(ext[OUT_KEY_VERSION], '0.0.3+dev')
-        with mock.patch('azure.cli.core.extension.operations.resolve_from_index', side_effect=NoExtensionCandidatesError()):
-            with self.assertRaises(CLIError) as err:
-                update_extension(self.cmd, MY_EXT_NAME)
-            self.assertTrue("No updates available for '{}'.".format(MY_EXT_NAME) in str(err.exception))
+        with mock.patch('azure.cli.core.extension.operations.resolve_from_index', side_effect=NoExtensionCandidatesError()), \
+                mock.patch('logging.Logger.warning', mock_log_warning):
+            update_extension(self.cmd, MY_EXT_NAME)
+        self.assertTrue("No updates available for '{}'.".format(MY_EXT_NAME) in logger_msgs[0])
 
     def test_update_extension_exception_in_update_and_rolled_back(self):
         add_extension(cmd=self.cmd, source=MY_EXT_SOURCE)

src/azure-cli-core/azure/cli/core/telemetry.py

@@ -280,8 +280,8 @@ def set_custom_properties(prop, name, value):
 
 @decorators.suppress_all_exceptions()
 def set_exception(exception, fault_type, summary=None):
-    if not summary:
-        _session.result_summary = summary
+    if not _session.result_summary:
+        _session.result_summary = _remove_cmd_chars(summary)
 
     _session.add_exception(exception, fault_type=fault_type, description=summary)
 

src/azure-cli-core/azure/cli/core/util.py

@@ -37,16 +37,6 @@
 
 _CHILDREN_RE = re.compile('(?i)/(?P<child_type>[^/]*)/(?P<child_name>[^/]*)')
 
-_PACKAGE_UPGRADE_INSTRUCTIONS = {"YUM": ("sudo yum update -y azure-cli", "https://aka.ms/doc/UpdateAzureCliYum"),
-                                 "ZYPPER": ("sudo zypper refresh && sudo zypper update -y azure-cli", "https://aka.ms/doc/UpdateAzureCliZypper"),
-                                 "DEB": ("sudo apt-get update && sudo apt-get install --only-upgrade -y azure-cli", "https://aka.ms/doc/UpdateAzureCliApt"),
-                                 "HOMEBREW": ("brew update && brew upgrade azure-cli", "https://aka.ms/doc/UpdateAzureCliHomebrew"),
-                                 "PIP": ("curl -L https://aka.ms/InstallAzureCli | bash", "https://aka.ms/doc/UpdateAzureCliLinux"),
-                                 "MSI": ("https://aka.ms/installazurecliwindows", "https://aka.ms/doc/UpdateAzureCliMsi"),
-                                 "DOCKER": ("docker pull mcr.microsoft.com/azure-cli", "https://aka.ms/doc/UpdateAzureCliDocker")}
-
-_GENERAL_UPGRADE_INSTRUCTION = 'Instructions can be found at https://aka.ms/doc/InstallAzureCli'
-
 _VERSION_CHECK_TIME = 'check_time'
 _VERSION_UPDATE_TIME = 'update_time'
 
@@ -171,25 +161,56 @@ def _update_latest_from_pypi(versions):
     return versions, success
 
 
+def get_latest_from_github(package_path='azure-cli'):
+    try:
+        import requests
+        git_url = "https://raw.githubusercontent.com/Azure/azure-cli/master/src/{}/setup.py".format(package_path)
+        response = requests.get(git_url, timeout=10)
+        if response.status_code != 200:
+            logger.info("Failed to fetch the latest version from '%s' with status code '%s' and reason '%s'",
+                        git_url, response.status_code, response.reason)
+            return None
+        for line in response.iter_lines():
+            txt = line.decode('utf-8', errors='ignore')
+            if txt.startswith('VERSION'):
+                match = re.search(r'VERSION = \"(.*)\"$', txt)
+                if match:
+                    return match.group(1)
+    except Exception as ex:  # pylint: disable=broad-except
+        logger.info("Failed to get the latest version from '%s'. %s", git_url, str(ex))
+        return None
+
+
+def _update_latest_from_github(versions):
+    if not check_connectivity(max_retries=0):
+        return versions, False
+    success = True
+    for pkg in ['azure-cli-core', 'azure-cli-telemetry']:
+        version = get_latest_from_github(pkg)
+        if not version:
+            success = False
+        else:
+            versions[pkg.replace(COMPONENT_PREFIX, '')]['pypi'] = version
+    versions[CLI_PACKAGE_NAME]['pypi'] = versions['core']['pypi']
+    return versions, success
+
+
 def get_cached_latest_versions(versions=None):
     """ Get the latest versions from a cached file"""
-    import os
     import datetime
-    from azure.cli.core._environment import get_config_dir
     from azure.cli.core._session import VERSIONS
 
     if not versions:
         versions = _get_local_versions()
 
-    VERSIONS.load(os.path.join(get_config_dir(), 'versionCheck.json'))
     if VERSIONS[_VERSION_UPDATE_TIME]:
         version_update_time = datetime.datetime.strptime(VERSIONS[_VERSION_UPDATE_TIME], '%Y-%m-%d %H:%M:%S.%f')
         if datetime.datetime.now() < version_update_time + datetime.timedelta(days=1):
             cache_versions = VERSIONS['versions']
             if cache_versions and cache_versions['azure-cli']['local'] == versions['azure-cli']['local']:
                 return cache_versions.copy(), True
 
-    versions, success = _update_latest_from_pypi(versions)
+    versions, success = _update_latest_from_github(versions)
     if success:
         VERSIONS['versions'] = versions
         VERSIONS[_VERSION_UPDATE_TIME] = str(datetime.datetime.now())
@@ -286,12 +307,9 @@ def get_az_version_json():
 
 
 def show_updates_available(new_line_before=False, new_line_after=False):
-    import os
     from azure.cli.core._session import VERSIONS
     import datetime
-    from azure.cli.core._environment import get_config_dir
 
-    VERSIONS.load(os.path.join(get_config_dir(), 'versionCheck.json'))
     if VERSIONS[_VERSION_CHECK_TIME]:
         version_check_time = datetime.datetime.strptime(VERSIONS[_VERSION_CHECK_TIME], '%Y-%m-%d %H:%M:%S.%f')
         if datetime.datetime.now() < version_check_time + datetime.timedelta(days=7):
@@ -314,34 +332,7 @@ def show_updates(updates_available):
         if in_cloud_console():
             warning_msg = 'You have %i updates available. They will be updated with the next build of Cloud Shell.'
         else:
-            warning_msg = 'You have %i updates available. Consider updating your CLI installation'
-            from azure.cli.core._environment import _ENV_AZ_INSTALLER
-            import os
-            installer = os.getenv(_ENV_AZ_INSTALLER)
-            instruction_msg = ''
-            if installer in _PACKAGE_UPGRADE_INSTRUCTIONS:
-                if installer == 'RPM':
-                    distname, _ = get_linux_distro()
-                    if not distname:
-                        instruction_msg = '. {}'.format(_GENERAL_UPGRADE_INSTRUCTION)
-                    else:
-                        distname = distname.lower().strip()
-                        if any(x in distname for x in ['centos', 'rhel', 'red hat', 'fedora']):
-                            installer = 'YUM'
-                        elif any(x in distname for x in ['opensuse', 'suse', 'sles']):
-                            installer = 'ZYPPER'
-                        else:
-                            instruction_msg = '. {}'.format(_GENERAL_UPGRADE_INSTRUCTION)
-                elif installer == 'PIP':
-                    system = platform.system()
-                    alternative_command = " or '{}' if you used our script for installation. Detailed instructions can be found at {}".format(_PACKAGE_UPGRADE_INSTRUCTIONS[installer][0], _PACKAGE_UPGRADE_INSTRUCTIONS[installer][1]) if system != 'Windows' else ''
-                    instruction_msg = " with 'pip install --upgrade azure-cli'{}".format(alternative_command)
-                if instruction_msg:
-                    warning_msg += instruction_msg
-                else:
-                    warning_msg += " with '{}'. Detailed instructions can be found at {}".format(_PACKAGE_UPGRADE_INSTRUCTIONS[installer][0], _PACKAGE_UPGRADE_INSTRUCTIONS[installer][1])
-            else:
-                warning_msg += '. {}'.format(_GENERAL_UPGRADE_INSTRUCTION)
+            warning_msg = "You have %i updates available. Consider updating your CLI installation with 'az upgrade'"
         logger.warning(warning_msg, updates_available)
     else:
         print('Your CLI is up-to-date.')
@@ -1036,3 +1027,20 @@ def is_guid(guid):
         return True
     except ValueError:
         return False
+
+
+def handle_version_update():
+    """Clean up information in local file that may be invalidated
+    because of a version update of Azure CLI
+    """
+    try:
+        from azure.cli.core._session import VERSIONS
+        from distutils.version import LooseVersion  # pylint: disable=import-error,no-name-in-module
+        from azure.cli.core import __version__
+        if not VERSIONS['versions']:
+            get_cached_latest_versions()
+        elif LooseVersion(VERSIONS['versions']['core']['local']) != LooseVersion(__version__):
+            VERSIONS['versions'] = {}
+            VERSIONS['update_time'] = ''
+    except Exception as ex:  # pylint: disable=broad-except
+        logger.warning(ex)

src/azure-cli/azure/cli/__main__.py

@@ -71,6 +71,43 @@ def cli_main(cli, args):
     except NameError:
         pass
 
+    try:
+        # check for new version auto-upgrade
+        if az_cli.config.getboolean('auto-upgrade', 'enable', False) and \
+                sys.argv[1] != 'upgrade' and (sys.argv[1] != 'extension' and sys.argv[2] != 'update'):
+            from azure.cli.core._session import VERSIONS  # pylint: disable=ungrouped-imports
+            from azure.cli.core.util import get_cached_latest_versions, _VERSION_UPDATE_TIME  # pylint: disable=ungrouped-imports
+            if VERSIONS[_VERSION_UPDATE_TIME]:
+                import datetime
+                version_update_time = datetime.datetime.strptime(VERSIONS[_VERSION_UPDATE_TIME], '%Y-%m-%d %H:%M:%S.%f')
+                if datetime.datetime.now() > version_update_time + datetime.timedelta(days=10):
+                    get_cached_latest_versions()
+                from distutils.version import LooseVersion
+                if LooseVersion(VERSIONS['versions']['core']['local']) < LooseVersion(VERSIONS['versions']['core']['pypi']):  # pylint: disable=line-too-long
+                    import subprocess
+                    import platform
+                    logger.warning("New Azure CLI version available. Running 'az upgrade' to update automatically.")
+                    update_all = az_cli.config.getboolean('auto-upgrade', 'all', True)
+                    prompt = az_cli.config.getboolean('auto-upgrade', 'prompt', True)
+                    cmd = ['az', 'upgrade', '--all', str(update_all)]
+                    if prompt:
+                        exit_code = subprocess.call(cmd, shell=platform.system() == 'Windows')
+                    else:
+                        import os
+                        devnull = open(os.devnull, 'w')
+                        cmd.append('-y')
+                        exit_code = subprocess.call(cmd, shell=platform.system() == 'Windows', stdout=devnull)
+                    if exit_code != 0:
+                        from knack.util import CLIError
+                        err_msg = "Auto upgrade failed with exit code {}".format(exit_code)
+                        logger.warning(err_msg)
+                        telemetry.set_exception(CLIError(err_msg), fault_type='auto-upgrade-failed')
+    except IndexError:
+        pass
+    except Exception as ex:  # pylint: disable=broad-except
+        logger.warning("Auto upgrade failed. %s", str(ex))
+        telemetry.set_exception(ex, fault_type='auto-upgrade-failed')
+
     telemetry.set_init_time_elapsed("{:.6f}".format(init_finish_time - start_time))
     telemetry.set_invoke_time_elapsed("{:.6f}".format(invoke_finish_time - init_finish_time))
     telemetry.conclude()

src/azure-cli/azure/cli/command_modules/extension/__init__.py

@@ -45,6 +45,7 @@ def validate_extension_add(namespace):
             g.show_command('show', 'show_extension_cmd')
             g.command('list-available', 'list_available_extensions_cmd', table_transformer=transform_extension_list_available)
             g.command('update', 'update_extension_cmd')
+            g.command('list-versions', 'list_versions_cmd')
 
         return self.command_table
 

src/azure-cli/azure/cli/command_modules/extension/_help.py

@@ -75,3 +75,11 @@
   - name: Update an extension by name and use pip proxy for dependencies
     text: az extension update --name anextension --pip-proxy https://user:pass@proxy.server:8080
 """
+
+helps['extension list-versions'] = """
+type: command
+short-summary: List available versions for an extension.
+examples:
+  - name: List available versions for an extension
+    text: az extension list-versions --name anextension
+"""

src/azure-cli/azure/cli/command_modules/extension/custom.py

@@ -6,7 +6,7 @@
 
 from azure.cli.core.extension.operations import (
     add_extension, remove_extension, list_extensions, show_extension,
-    list_available_extensions, update_extension)
+    list_available_extensions, update_extension, list_versions)
 
 logger = get_logger(__name__)
 
@@ -37,3 +37,7 @@ def update_extension_cmd(cmd, extension_name, index_url=None, pip_extra_index_ur
 
 def list_available_extensions_cmd(index_url=None, show_details=False):
     return list_available_extensions(index_url=index_url, show_details=show_details)
+
+
+def list_versions_cmd(extension_name, index_url=None):
+    return list_versions(extension_name, index_url=index_url)

src/azure-cli/azure/cli/command_modules/util/_help.py

@@ -40,3 +40,8 @@
 type: command
 short-summary: Show the versions of Azure CLI modules and extensions in JSON format by default or format configured by --output
 """
+
+helps['upgrade'] = """
+type: command
+short-summary: Upgrade Azure CLI and extensions
+"""

src/azure-cli/azure/cli/command_modules/util/_params.py

@@ -6,7 +6,7 @@
 
 # pylint: disable=line-too-long
 def load_arguments(self, _):
-    from azure.cli.core.commands.parameters import get_enum_type
+    from azure.cli.core.commands.parameters import get_enum_type, get_three_state_flag
 
     with self.argument_context('rest') as c:
         c.argument('method', options_list=['--method', '-m'],
@@ -33,3 +33,7 @@ def load_arguments(self, _):
                         'the service. The token will be placed in the Authorization header. By default, '
                         'CLI can figure this out based on --url argument, unless you use ones not in the list '
                         'of "az cloud show --query endpoints"')
+
+    with self.argument_context('upgrade') as c:
+        c.argument('update_all', options_list=['--all'], arg_type=get_three_state_flag(), help='Enable updating extensions as well.', default='true')
+        c.argument('yes', options_list=['--yes', '-y'], action='store_true', help='Do not prompt for checking release notes.')

src/azure-cli/azure/cli/command_modules/util/commands.py

@@ -11,3 +11,6 @@ def load_command_table(self, _):
 
     with self.command_group('') as g:
         g.custom_command('version', 'show_version')
+
+    with self.command_group('') as g:
+        g.custom_command('upgrade', 'upgrade_version', is_experimental=True)