Skip to content

Commit 17be5e8

Browse files
blink1073eramongodb
blink1073
and
eramongodb
authored
DRIVERS-3032 Add retries to download attempts (#612)
Co-authored-by: Ezra Chung <88335979+eramongodb@users.noreply.github.com>
1 parent 1513f49 commit 17be5e8

File tree

4 files changed

+89
-38
lines changed

4 files changed

+89
-38
lines changed

‎.evergreen/mongodl.py‎

Lines changed: 39 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import sys
2929
import tarfile
3030
import textwrap
31+
import time
3132
import urllib.error
3233
import urllib.request
3334
import warnings
@@ -317,6 +318,26 @@ def mdb_version_rapid(version: str) -> bool:
317318
return tup[1] > 0
318319

319320

321+
class DownloadRetrier:
322+
"""Class that handles retry logic. It performs exponential backoff with a maximum delay of 10 minutes between retry attempts."""
323+
324+
def __init__(self, retries: int) -> None:
325+
self.retries = retries
326+
self.attempt = 0
327+
assert self.retries >= 0
328+
329+
def retry(self) -> bool:
330+
if self.attempt >= self.retries:
331+
return False
332+
self.attempt += 1
333+
LOGGER.warning(
334+
f"Download attempt failed, retrying attempt {self.attempt} of {self.retries}"
335+
)
336+
ten_minutes = 600
337+
time.sleep(min(2 ** (self.attempt - 1), ten_minutes))
338+
return True
339+
340+
320341
class CacheDB:
321342
"""
322343
Abstract a mongodl cache SQLite database.
@@ -832,6 +853,7 @@ def _dl_component(
832853
test: bool,
833854
no_download: bool,
834855
latest_build_branch: "str|None",
856+
retries: int,
835857
) -> ExpandResult:
836858
LOGGER.info(f"Download {component} {version}-{edition} for {target}-{arch}")
837859
if version in ("latest-build", "latest"):
@@ -859,12 +881,23 @@ def _dl_component(
859881
cache, version, target, arch, edition, component
860882
)
861883

884+
# This must go to stdout to be consumed by the calling program.
885+
print(dl_url)
886+
LOGGER.info("Download url: %s", dl_url)
887+
862888
if no_download:
863-
# This must go to stdout to be consumed by the calling program.
864-
print(dl_url)
865889
return None
866-
cached = cache.download_file(dl_url).path
867-
return _expand_archive(cached, out_dir, pattern, strip_components, test=test)
890+
891+
retrier = DownloadRetrier(retries)
892+
while True:
893+
try:
894+
cached = cache.download_file(dl_url).path
895+
return _expand_archive(
896+
cached, out_dir, pattern, strip_components, test=test
897+
)
898+
except Exception:
899+
if not retrier.retry():
900+
raise
868901

869902

870903
def _pathjoin(items: "Iterable[str]") -> PurePath:
@@ -1145,6 +1178,7 @@ def main(argv=None):
11451178
'download the with "--version=latest-build"',
11461179
metavar="BRANCH_NAME",
11471180
)
1181+
dl_grp.add_argument("--retries", help="The number of times to retry", default=0)
11481182
args = parser.parse_args(argv)
11491183
cache = Cache.open_in(args.cache_dir)
11501184
cache.refresh_full_json()
@@ -1184,6 +1218,7 @@ def main(argv=None):
11841218
test=args.test,
11851219
no_download=args.no_download,
11861220
latest_build_branch=args.latest_build_branch,
1221+
retries=int(args.retries),
11871222
)
11881223
if result is ExpandResult.Empty and args.empty_is_error:
11891224
sys.exit(1)

‎.evergreen/mongosh_dl.py‎

Lines changed: 30 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,13 @@
2323
HERE = Path(__file__).absolute().parent
2424
sys.path.insert(0, str(HERE))
2525
from mongodl import LOGGER as DL_LOGGER
26-
from mongodl import SSL_CONTEXT, ExpandResult, _expand_archive, infer_arch
26+
from mongodl import (
27+
SSL_CONTEXT,
28+
DownloadRetrier,
29+
ExpandResult,
30+
_expand_archive,
31+
infer_arch,
32+
)
2733

2834

2935
def _get_latest_version():
@@ -75,6 +81,7 @@ def _download(
7581
strip_components: int,
7682
test: bool,
7783
no_download: bool,
84+
retries: int,
7885
) -> int:
7986
LOGGER.info(f"Download {version} mongosh for {target}-{arch}")
8087
if version == "latest":
@@ -96,24 +103,31 @@ def _download(
96103
dl_url = f"https://downloads.mongodb.com/compass/mongosh-{version}-{target}-{arch}{suffix}"
97104
# This must go to stdout to be consumed by the calling program.
98105
print(dl_url)
106+
LOGGER.info("Download url: %s", dl_url)
99107

100108
if no_download:
101109
return ExpandResult.Okay
102110

103111
req = urllib.request.Request(dl_url)
104-
resp = urllib.request.urlopen(req)
105-
106-
with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as fp:
107-
buf = resp.read(1024 * 1024 * 4)
108-
while buf:
109-
fp.write(buf)
110-
buf = resp.read(1024 * 1024 * 4)
111-
fp.close()
112-
resp = _expand_archive(
113-
Path(fp.name), out_dir, pattern, strip_components, test=test
114-
)
115-
os.remove(fp.name)
116-
return resp
112+
retrier = DownloadRetrier(retries)
113+
while True:
114+
try:
115+
resp = urllib.request.urlopen(req)
116+
with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as fp:
117+
four_mebibytes = 1024 * 1024 * 4
118+
buf = resp.read(four_mebibytes)
119+
while buf:
120+
fp.write(buf)
121+
buf = resp.read(four_mebibytes)
122+
fp.close()
123+
resp = _expand_archive(
124+
Path(fp.name), out_dir, pattern, strip_components, test=test
125+
)
126+
os.remove(fp.name)
127+
return resp
128+
except Exception:
129+
if not retrier.retry():
130+
raise
117131

118132

119133
def main(argv=None):
@@ -189,6 +203,7 @@ def main(argv=None):
189203
help="Do not extract or place any files/directories. "
190204
"Only print what will be extracted without placing any files.",
191205
)
206+
dl_grp.add_argument("--retries", help="The number of times to retry", default=0)
192207
args = parser.parse_args(argv)
193208

194209
target = args.target
@@ -214,6 +229,7 @@ def main(argv=None):
214229
strip_components=args.strip_components,
215230
test=args.test,
216231
no_download=args.no_download,
232+
retries=int(args.retries),
217233
)
218234
if result is ExpandResult.Empty:
219235
sys.exit(1)

‎.evergreen/orchestration/drivers_orchestration.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,7 @@ def run(opts):
194194
version = opts.version
195195
cache_dir = DRIVERS_TOOLS / ".local/cache"
196196
cache_dir_str = cache_dir.as_posix()
197-
default_args = f"--out {mdb_binaries_str} --cache-dir {cache_dir_str}"
197+
default_args = f"--out {mdb_binaries_str} --cache-dir {cache_dir_str} --retries 5"
198198
if opts.quiet:
199199
default_args += " -q"
200200
elif opts.verbose:
@@ -243,7 +243,7 @@ def run(opts):
243243
expansion_sh.write_text(crypt_text.replace(": ", "="))
244244

245245
# Download mongosh
246-
args = f"--out {mdb_binaries_str} --strip-path-components 2"
246+
args = f"--out {mdb_binaries_str} --strip-path-components 2 --retries 5"
247247
if opts.verbose:
248248
args += " -v"
249249
elif opts.quiet:

‎.evergreen/tests/test-cli.sh‎

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -24,24 +24,24 @@ else
2424
DOWNLOAD_DIR=$(cygpath -m $DOWNLOAD_DIR)
2525
fi
2626

27-
./mongodl --edition enterprise --version 7.0 --component archive --test
28-
./mongodl --edition enterprise --version 7.0 --component cryptd --out ${DOWNLOAD_DIR} --strip-path-components 1
27+
./mongodl --edition enterprise --version 7.0 --component archive --test --retries 5
28+
./mongodl --edition enterprise --version 7.0 --component cryptd --out ${DOWNLOAD_DIR} --strip-path-components 1 --retries 5
2929
./mongosh-dl --no-download
3030
./mongosh-dl --version 2.1.1 --no-download
3131

3232
export PATH="${DOWNLOAD_DIR}/bin:$PATH"
3333
if [ "${OS:-}" != "Windows_NT" ]; then
34-
./mongosh-dl --version 2.1.1 --out ${DOWNLOAD_DIR} --strip-path-components 1
34+
./mongosh-dl --version 2.1.1 --out ${DOWNLOAD_DIR} --strip-path-components 1 --retries 5
3535
chmod +x ./mongodl_test/bin/mongosh
3636
./mongodl_test/bin/mongosh --version
3737
else
38-
./mongosh-dl --version 2.1.1 --out ${DOWNLOAD_DIR} --strip-path-components 1
38+
./mongosh-dl --version 2.1.1 --out ${DOWNLOAD_DIR} --strip-path-components 1 --retries 5
3939
fi
4040

4141
# Ensure that we can use a downloaded mongodb directory.
4242
rm -rf ${DOWNLOAD_DIR}
4343
bash install-cli.sh "$(pwd)/orchestration"
44-
./mongodl --edition enterprise --version 7.0 --component archive --out ${DOWNLOAD_DIR} --strip-path-components 2
44+
./mongodl --edition enterprise --version 7.0 --component archive --out ${DOWNLOAD_DIR} --strip-path-components 2 --retries 5
4545
./orchestration/drivers-orchestration run --existing-binaries-dir=${DOWNLOAD_DIR}
4646
${DOWNLOAD_DIR}/mongod --version | grep v7.0
4747
./orchestration/drivers-orchestration stop
@@ -56,19 +56,19 @@ fi
5656
export VALIDATE_DISTROS=1
5757
./mongodl --list
5858
./mongodl --edition enterprise --version 7.0.6 --component archive --no-download
59-
./mongodl --edition enterprise --version 3.6 --component archive --test
60-
./mongodl --edition enterprise --version 4.0 --component archive --test
61-
./mongodl --edition enterprise --version 4.2 --component archive --test
62-
./mongodl --edition enterprise --version 4.4 --component archive --test
63-
./mongodl --edition enterprise --version 5.0 --component archive --test
64-
./mongodl --edition enterprise --version 6.0 --component crypt_shared --test
65-
./mongodl --edition enterprise --version 8.0 --component archive --test
66-
./mongodl --edition enterprise --version rapid --component archive --test
67-
./mongodl --edition enterprise --version latest --component archive --out ${DOWNLOAD_DIR}
68-
./mongodl --edition enterprise --version latest-build --component archive --test
69-
./mongodl --edition enterprise --version latest-release --component archive --test
70-
./mongodl --edition enterprise --version v6.0-perf --component cryptd --test
71-
./mongodl --edition enterprise --version v8.0-perf --component cryptd --test
59+
./mongodl --edition enterprise --version 3.6 --component archive --test --retries 5
60+
./mongodl --edition enterprise --version 4.0 --component archive --test --retries 5
61+
./mongodl --edition enterprise --version 4.2 --component archive --test --retries 5
62+
./mongodl --edition enterprise --version 4.4 --component archive --test --retries 5
63+
./mongodl --edition enterprise --version 5.0 --component archive --test --retries 5
64+
./mongodl --edition enterprise --version 6.0 --component crypt_shared --test --retries 5
65+
./mongodl --edition enterprise --version 8.0 --component archive --test --retries 5
66+
./mongodl --edition enterprise --version rapid --component archive --test --retries 5
67+
./mongodl --edition enterprise --version latest --component archive --out ${DOWNLOAD_DIR} --retries 5
68+
./mongodl --edition enterprise --version latest-build --component archive --test --retries 5
69+
./mongodl --edition enterprise --version latest-release --component archive --test --retries 5
70+
./mongodl --edition enterprise --version v6.0-perf --component cryptd --test --retries 5
71+
./mongodl --edition enterprise --version v8.0-perf --component cryptd --test --retries 5
7272

7373
popd
7474
make -C ${DRIVERS_TOOLS} test

0 commit comments

Comments
 (0)