Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[cherry-pick] remove _wget (#61356) #61569

Merged
merged 1 commit into from
Feb 5, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion python/paddle/hapi/hub.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,6 @@ def _get_cache_or_reload(repo, force_reload, verbose=True, source='github'):
hub_dir,
check_exist=not force_reload,
decompress=False,
method=('wget' if source == 'gitee' else 'get'),
)
shutil.move(fpath, cached_file)

Expand Down
38 changes: 1 addition & 37 deletions python/paddle/utils/download.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,11 @@
import hashlib
import os
import os.path as osp
import shlex
import shutil
import subprocess
import sys
import tarfile
import time
import zipfile
from urllib.parse import urlparse

import httpx

Expand Down Expand Up @@ -198,40 +195,7 @@ def _get_download(url, fullname):
return False


def _wget_download(url: str, fullname: str):
try:
assert urlparse(url).scheme in (
'http',
'https',
), 'Only support https and http url'
# using wget to download url
tmp_fullname = shlex.quote(fullname + "_tmp")
url = shlex.quote(url)
# –user-agent
command = f'wget -O {tmp_fullname} -t {DOWNLOAD_RETRY_LIMIT} {url}'
subprc = subprocess.Popen(
command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
_ = subprc.communicate()

if subprc.returncode != 0:
raise RuntimeError(
f'{command} failed. Please make sure `wget` is installed or {url} exists'
)

shutil.move(tmp_fullname, fullname)

except Exception as e: # requests.exceptions.ConnectionError
logger.info(f"Downloading {url} failed with exception {str(e)}")
return False

return fullname


_download_methods = {
'get': _get_download,
'wget': _wget_download,
}
_download_methods = {'get': _get_download}


def _download(url, path, md5sum=None, method='get'):
Expand Down
15 changes: 1 addition & 14 deletions test/legacy_test/test_download.py
Original file line number Diff line number Diff line change
Expand Up @@ -120,14 +120,6 @@ def test_retry_exception(
'./test',
)

def test_wget_download_error(
self,
):
with self.assertRaises(RuntimeError):
from paddle.utils.download import _download

_download('www.baidu', './test', method='wget')

def test_download_methods(
self,
):
Expand All @@ -136,14 +128,9 @@ def test_download_methods(
"https://paddle-hapi.bj.bcebos.com/unittest/files.zip",
]

import sys

from paddle.utils.download import _download

if sys.platform == 'linux':
methods = ['wget', 'get']
else:
methods = ['get']
methods = ['get']

for url in urls:
for method in methods:
Expand Down