From 3c0e036b449fcc4544f153aaace4a97393848115 Mon Sep 17 00:00:00 2001 From: lyuwenyu Date: Tue, 30 Jan 2024 17:43:01 +0800 Subject: [PATCH 1/3] remove _wget --- python/paddle/hapi/hub.py | 1 - python/paddle/utils/download.py | 31 ------------------------------- 2 files changed, 32 deletions(-) diff --git a/python/paddle/hapi/hub.py b/python/paddle/hapi/hub.py index 254e3a833f0db..4fa1d59cc9bc5 100644 --- a/python/paddle/hapi/hub.py +++ b/python/paddle/hapi/hub.py @@ -117,7 +117,6 @@ def _get_cache_or_reload(repo, force_reload, verbose=True, source='github'): hub_dir, check_exist=not force_reload, decompress=False, - method=('wget' if source == 'gitee' else 'get'), ) shutil.move(fpath, cached_file) diff --git a/python/paddle/utils/download.py b/python/paddle/utils/download.py index 30f02a20b969b..7a40b8e954ac7 100644 --- a/python/paddle/utils/download.py +++ b/python/paddle/utils/download.py @@ -198,39 +198,8 @@ def _get_download(url, fullname): return False -def _wget_download(url: str, fullname: str): - try: - assert urlparse(url).scheme in ( - 'http', - 'https', - ), 'Only support https and http url' - # using wget to download url - tmp_fullname = shlex.quote(fullname + "_tmp") - url = shlex.quote(url) - # –user-agent - command = f'wget -O {tmp_fullname} -t {DOWNLOAD_RETRY_LIMIT} {url}' - subprc = subprocess.Popen( - command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - _ = subprc.communicate() - - if subprc.returncode != 0: - raise RuntimeError( - f'{command} failed. Please make sure `wget` is installed or {url} exists' - ) - - shutil.move(tmp_fullname, fullname) - - except Exception as e: # requests.exceptions.ConnectionError - logger.info(f"Downloading {url} failed with exception {str(e)}") - return False - - return fullname - - _download_methods = { 'get': _get_download, - 'wget': _wget_download, } From 4490913b41c7d150838b9820f8e20a523c5c8e76 Mon Sep 17 00:00:00 2001 From: lyuwenyu Date: Tue, 30 Jan 2024 19:24:17 +0800 Subject: [PATCH 2/3] remove _wget --- python/paddle/utils/download.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/python/paddle/utils/download.py b/python/paddle/utils/download.py index 7a40b8e954ac7..e0cd17662d7f1 100644 --- a/python/paddle/utils/download.py +++ b/python/paddle/utils/download.py @@ -15,14 +15,11 @@ import hashlib import os import os.path as osp -import shlex import shutil -import subprocess import sys import tarfile import time import zipfile -from urllib.parse import urlparse import httpx @@ -198,9 +195,7 @@ def _get_download(url, fullname): return False -_download_methods = { - 'get': _get_download, -} +_download_methods = {'get': _get_download} def _download(url, path, md5sum=None, method='get'): From 035d79f976c6d6e9efe2a46ab878eb6a91d79531 Mon Sep 17 00:00:00 2001 From: lyuwenyu Date: Thu, 1 Feb 2024 12:05:12 +0800 Subject: [PATCH 3/3] remove wget test --- test/legacy_test/test_download.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/test/legacy_test/test_download.py b/test/legacy_test/test_download.py index 742c4b2a65190..da25a3021a31e 100644 --- a/test/legacy_test/test_download.py +++ b/test/legacy_test/test_download.py @@ -120,14 +120,6 @@ def test_retry_exception( './test', ) - def test_wget_download_error( - self, - ): - with self.assertRaises(RuntimeError): - from paddle.utils.download import _download - - _download('www.baidu', './test', method='wget') - def test_download_methods( self, ): @@ -136,14 +128,9 @@ def test_download_methods( "https://paddle-hapi.bj.bcebos.com/unittest/files.zip", ] - import sys - from paddle.utils.download import _download - if sys.platform == 'linux': - methods = ['wget', 'get'] - else: - methods = ['get'] + methods = ['get'] for url in urls: for method in methods: