cyclecloud-cli-installer install.sh cannot resolve URL

[mjones-mmu]

My environment:

Platform = Azure Virtual Machine
OS = AlmaLinux 8.10
CycleCloud Vers = 8.7.0-3349

Reproducing the problem:

[root@server ~]# dnf install cyclecloud8

[root@server ~]# cd /opt/cycle_server/tools

[root@server ~]# unzip cyclecloud-cli.zip

[root@server ~]#  ln -s /bin/python3.8 /bin/python3

[root@server ~]#  sh /opt/cycle_server/tools/cyclecloud-cli-installer/install.sh --system

Error output:

Installation Directory: /usr/local/cyclecloud-cli
Scripts Directory: /usr/local/bin
Creating virtual environment...
Installing...
Traceback (most recent call last):
  File "/usr/lib64/python3.8/urllib/request.py", line 1354, in do_open
    h.request(req.get_method(), req.selector, req.data, headers,
  File "/usr/lib64/python3.8/http/client.py", line 1256, in request
    self._send_request(method, url, body, headers, encode_chunked)
  File "/usr/lib64/python3.8/http/client.py", line 1302, in _send_request
    self.endheaders(body, encode_chunked=encode_chunked)
  File "/usr/lib64/python3.8/http/client.py", line 1251, in endheaders
    self._send_output(message_body, encode_chunked=encode_chunked)
  File "/usr/lib64/python3.8/http/client.py", line 1011, in _send_output
    self.send(msg)
  File "/usr/lib64/python3.8/http/client.py", line 951, in send
    self.connect()
  File "/usr/lib64/python3.8/http/client.py", line 1418, in connect
    super().connect()
  File "/usr/lib64/python3.8/http/client.py", line 922, in connect
    self.sock = self._create_connection(
  File "/usr/lib64/python3.8/socket.py", line 787, in create_connection
    for res in getaddrinfo(host, port, 0, SOCK_STREAM):
  File "/usr/lib64/python3.8/socket.py", line 918, in getaddrinfo
    for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno -2] Name or service not known

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/cycle_server/tools/cyclecloud-cli-installer/install.py", line 271, in <module>
    main()
  File "/opt/cycle_server/tools/cyclecloud-cli-installer/install.py", line 263, in main
    fetch_azcopy(get_venv_bin_path(venv_dir))
  File "/opt/cycle_server/tools/cyclecloud-cli-installer/install.py", line 70, in fetch_azcopy
    urllib.request.urlretrieve(_AZ_COPY_URL, archive_file)
  File "/usr/lib64/python3.8/urllib/request.py", line 247, in urlretrieve
    with contextlib.closing(urlopen(url, data)) as fp:
  File "/usr/lib64/python3.8/urllib/request.py", line 222, in urlopen
    return opener.open(url, data, timeout)
  File "/usr/lib64/python3.8/urllib/request.py", line 525, in open
    response = self._open(req, data)
  File "/usr/lib64/python3.8/urllib/request.py", line 542, in _open
    result = self._call_chain(self.handle_open, protocol, protocol +
  File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
    result = func(*args)
  File "/usr/lib64/python3.8/urllib/request.py", line 1397, in https_open
    return self.do_open(http.client.HTTPSConnection, req,
  File "/usr/lib64/python3.8/urllib/request.py", line 1357, in do_open
    raise URLError(err)
urllib.error.URLError: <urlopen error [Errno -2] Name or service not known>

My investigation:

• Upon inspecting the python installer script "/opt/cycle_server/tools/cyclecloud-cli-installer/install.py" I found the URL "https://azcopyvnext.azureedge.net/releases/release-10.27.1-20241113/azcopy_linux_amd64_10.27.1.tar.gz" which I am unable to resolve by any means on any system.
• I believe this is linked to the recent edgio bankruptcy and the CDN being shutdown.
• Similar issue - upload to Azure storage broken due to azure-cli bug skypilot-org/skypilot#4568

Workaround :

• Update /opt/cycle_server/tools/cyclecloud-cli-installer/install.py to use https://aka.ms/downloadazcopy-v10-linux

Closing this ticket as won't resolve as I'm fairly certain it's not something to be resolved here, but may impact many users nevertheless.

[diodfr]

We are facing the same issue. Could you explain why you closed it ?

[mjones-mmu]

We are facing the same issue. Could you explain why you closed it ?

I opened the issue by mistake here but I don't think the contributors of this github are responsible for resolving the package issue with cyclecloud8 as it is Microsoft themselves who manage the repository it is sourced from.

Instead I opened a ticket with Microsoft Azure support on the matter who confirmed this to be the case, they suggested that the package owners were already aware of the problem and whilst they won't commit to a full release date containing the fix, they have already made a pre-release version available here https://packages.microsoft.com/yumrepos/cyclecloud-insiders/Packages/c/cyclecloud8-8.7.1-3353.x86_64.rpm.

In the meantime I have successfully tested implementing a workaround to the problem (for Linux distro's only):

1. Edit /opt/cycle_server/tools/cyclecloud-cli-installer/install.py
2. See my custom script below which replaces the URL and also implements safe extraction for tar files which eliminates a warning in Python38

#!/usr/bin/env python

REQUIRED_PYTHON_VERSION = (3, 8, 0)

import sys

def assert_python_version():
    vi = sys.version_info
    current_version = (vi[0], vi[1], vi[2])
    if current_version < REQUIRED_PYTHON_VERSION:
        sys.stderr.write("CycleCloud requires Python %s.%s.%s or later, found: %s.%s.%s\n" % (REQUIRED_PYTHON_VERSION + current_version))
        sys.stdout.write("Exiting now.\n")
        sys.stderr.flush()
        sys.stdout.flush()
        sys.exit(1)

assert_python_version()

import argparse
import logging
import os
import platform
import shutil
import subprocess
import tarfile
import tempfile
import urllib.request
import venv
import zipfile
import glob
import hashlib
import tarfile
import warnings

# Suppress the specific warning
warnings.filterwarnings("ignore", category=RuntimeWarning, message=".*tarfile.*")

_IS_WINDOWS = platform.system().lower().startswith("win")
_IS_OSX = sys.platform == 'darwin'
_IS_LINUX = not _IS_WINDOWS and not _IS_OSX

if _IS_LINUX:
    _AZ_COPY_URL = https://aka.ms/downloadazcopy-v10-linux
    _AZ_COPY_CHECKSUM = "b66b35c564b88c2e89f0c6b179ea52c42f1dd35924e2f0ebbbc5ff39c94f1d2c"
elif _IS_WINDOWS:
    _AZ_COPY_URL = https://azcopyvnext.azureedge.net/releases/release-10.27.1-20241113/azcopy_windows_amd64_10.27.1.zip
    _AZ_COPY_CHECKSUM = "d0cfe7c3682b960f1ff70f4f763f8ac89a13bed91866ee4cea1150daf2cd8aa0"
elif _IS_OSX:
    _AZ_COPY_URL = https://azcopyvnext.azureedge.net/releases/release-10.27.1-20241113/azcopy_darwin_amd64_10.27.1.zip
    _AZ_COPY_CHECKSUM = "a72d4d9515fed4c516963e9ce0fe95c37bdf34d9f060b5b2d7dc8705e382d24d"

def print_now(msg, err=False):
    if err:
        stream = sys.stderr
    else:
        stream = sys.stdout

    stream.write("%s\n" % msg)
    stream.flush()

def do_error(msg, do_exit=True):
    msg = "ERROR: " + msg
    print_now(msg, err=True)
    if do_exit:
        print_now('Exiting now')
        sys.exit(1)

def safe_extract(tar, path=".", members=None, *, numeric_owner=False):
    for member in tar.getmembers():
        member_path = os.path.join(path, member.name)
        if not os.path.commonprefix([path, member_path]) == os.path.abspath(path):
            raise Exception("Attempted Path Traversal in Tar File")
    tar.extractall(path, members, numeric_owner=numeric_owner)

def fetch_azcopy(destination):
    with tempfile.TemporaryDirectory() as temp_dir:
        archive_file = os.path.join(temp_dir, "archivefile")
        urllib.request.urlretrieve(_AZ_COPY_URL, archive_file)

        checksum = hashlib.sha256()
        with open(archive_file, "rb") as f:
            for block in iter(lambda: f.read(8192), b""):
                checksum.update(block)
        if  checksum.hexdigest().lower() != _AZ_COPY_CHECKSUM:
            raise Exception("Checksum mismatch for downloaded file: " + _AZ_COPY_URL)

        if _IS_WINDOWS:
            with zipfile.ZipFile(archive_file, "r") as f:
                f.extractall(path=temp_dir)
        else:
            with tarfile.open(archive_file, 'r:gz') as f:
                safe_extract(f, path=temp_dir)
        binaries = glob.glob(os.path.join(temp_dir, "*", "azcopy*"))
        if len(binaries) == 0:
            raise Exception("No azcopy binary found in archive: " + _AZ_COPY_URL)
        elif len(binaries) > 1:
            raise Exception("Multiple azcopy binaries found in archive: " + _AZ_COPY_URL)
        os.chmod(binaries[0], 0o755)
        shutil.move(binaries[0], destination)

def configure_logging(verbose):
    level = logging.WARN
    if verbose:
        level = logging.DEBUG

    logging.basicConfig(level=level)

def assert_can_install_systemwide():
    '''
    Assert that the user is executing this script as root/sudo/admin
    '''

    if _IS_WINDOWS:
        import ctypes
        if not ctypes.windll.shell32.IsUserAnAdmin():
            do_error('You must execute this script as an administrator to use the --system option')
    else:
        if not os.geteuid() == 0:
            do_error('You must execute this script with sudo or as root to use the --system option')

def create_virtualenv(venv_dir, assumeyes):
    if os.path.isfile(venv_dir):
        do_error("The installation directory is already occupied by a file.")

    elif os.path.isdir(venv_dir):
        if not assumeyes:
            sys.stdout.write("An existing installation is present at: %s\nAre you sure you want to replace it? [Y/n] " % venv_dir)
            sys.stdout.flush()
            v = sys.stdin.readline().strip()
        else:
            v = "yes"

        if v.lower() in ["", "y", "yes"]:
            shutil.rmtree(venv_dir)
        else:
            print_now("Installation Canceled")
            sys.exit(1)

    parentdir = os.path.dirname(venv_dir)
    if not os.path.exists(parentdir):
        logging.debug('Creating directory %s', parentdir)
        os.makedirs(parentdir)

    print_now('Creating virtual environment...')
    venv.create(venv_dir, with_pip=True)

def get_venv_bin_path(venv_dir):
    # the subdirectory w/ executables varies between windows and everything else
    subdirectory = 'bin'
    if _IS_WINDOWS:
        subdirectory = 'Scripts'

    return os.path.join(venv_dir, subdirectory)

def find_packages(package_dir):
    files = [os.path.join(package_dir, f) for f in os.listdir(package_dir)]
    packages = [p for p in files if os.path.isfile(p)]
    logging.debug('Found packages %s', packages)
    return packages

def install_packages(venv_dir, package_dir):
    venv_bin_dir = get_venv_bin_path(venv_dir)
    pip_path = os.path.join(venv_bin_dir, 'pip')
    install_cmd = [pip_path, 'install']
    packages = find_packages(package_dir)
    install_cmd.extend(packages)
    print_now("Installing...")
    logging.debug('Executing command %s', ' '.join(install_cmd))
    p = subprocess.Popen(install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    _, stderr = p.communicate()
    if p.returncode != 0:
        do_error('Installation of packages failed with stderr: %s' % stderr)
    else:
        logging.debug('Successfully installed packages %s', packages)

def setup_bin_dir(venv_dir, bin_dir, system_install):
    print_now("Creating script references...")
    venv_bin_dir = get_venv_bin_path(venv_dir)
    if not os.path.exists(bin_dir):
        os.makedirs(bin_dir)

    if _IS_WINDOWS:
        copy_script(venv_bin_dir, bin_dir, 'cyclecloud.exe')

        script_path = os.path.join(os.path.dirname(__file__), "update-path.ps1")

        target_env = "Machine" if system_install else "User"

        cmd = 'powershell.exe -file "%s" %s "%s"' % (script_path, target_env, bin_dir)
        subprocess.check_call(cmd, shell=True)

    else:
        link_script(venv_bin_dir, bin_dir, 'cyclecloud')

        if bin_dir not in os.environ['PATH']:
            print_now("'%s' not found in your PATH environment variable. Make sure to update it."
                      % bin_dir, err=True)

def copy_script(venv_bin_dir, bin_dir, executable):
    source = os.path.join(venv_bin_dir, executable)
    target = os.path.join(bin_dir, executable)

    if os.path.exists(target):
        logging.debug('Script %s exists, deleting', target)
        os.remove(target)

    shutil.copyfile(source, target)

def link_script(venv_bin_dir, bin_dir, executable):
    source = os.path.join(venv_bin_dir, executable)
    target = os.path.join(bin_dir, executable)

    if os.path.islink(target):
        logging.debug('Link %s exists, deleting', target)
        os.remove(target)
    elif os.path.exists(target):
        do_error('Path %s exists and is not a symlink, cannot update' % target)

    logging.debug('Symlinking %s to %s', target, source)
    os.symlink(source, target)
    os.chmod(target, 0o755)

def main():
    parser = argparse.ArgumentParser(description='Installs the CycleCloud CLI')
    parser.add_argument('--installdir', default='~/.cycle/cli',
                        help='path to install the cli tools')
    parser.add_argument('--system', action='store_true', default=False,
                        help='install executables so that they are available to all users')
    parser.add_argument('-v', '--verbose', action='store_true', default=False,
                        help='print logging information')
    parser.add_argument('-y', '--assumeyes', action="store_true",
                       help='assume yes for any confirmation prompts')
    args = parser.parse_args()

    configure_logging(args.verbose)

    if args.system:
        assert_can_install_systemwide()
        if _IS_WINDOWS:
            venv_dir = 'C:\\Program Files\\CycleCloud-Cli'
            bin_dir = venv_dir + '\\bin'
        else:
            venv_dir = '/usr/local/cyclecloud-cli'
            bin_dir = '/usr/local/bin'
    else:
        venv_dir = os.path.abspath(os.path.expanduser(args.installdir))
        if _IS_WINDOWS:
            bin_dir = venv_dir + '\\bin'
        else:
            bin_dir = os.path.abspath(os.path.expanduser('~/bin'))

    package_dir = os.path.join(os.path.dirname(__file__), 'packages')

    print_now('Installation Directory: %s' % venv_dir)
    print_now('Scripts Directory: %s' % bin_dir)

    create_virtualenv(venv_dir, args.assumeyes)

    install_packages(venv_dir, package_dir)

    fetch_azcopy(get_venv_bin_path(venv_dir))

    setup_bin_dir(venv_dir, bin_dir, args.system)

    print_now("Installation Complete")

if __name__ == '__main__':
    main()