Skip to content

Commit

Permalink
Switch to .xz by default for SDK downloads
Browse files Browse the repository at this point in the history
This is a bit of a hack but I can't think of another way to do it.
Basically when downloading SDKs, we first try the new `.xz` extension.
If that fails, we fall back to the old `.tbz2`.  Both these first two
download attempts we run in "silent" mode.  If both of them fail we
re-run the original request in non-silent mode so that the error message
will always contain the original `.xz` extension.

See #1235
  • Loading branch information
sbc100 committed Sep 21, 2023
1 parent 93360d3 commit 6d11425
Show file tree
Hide file tree
Showing 2 changed files with 32 additions and 12 deletions.
36 changes: 28 additions & 8 deletions emsdk.py
Original file line number Diff line number Diff line change
Expand Up @@ -672,7 +672,8 @@ def get_download_target(url, dstpath, filename_prefix=''):

# On success, returns the filename on the disk pointing to the destination file that was produced
# On failure, returns None.
def download_file(url, dstpath, download_even_if_exists=False, filename_prefix=''):
def download_file(url, dstpath, download_even_if_exists=False,
filename_prefix='', silent=False):
debug_print('download_file(url=' + url + ', dstpath=' + dstpath + ')')
file_name = get_download_target(url, dstpath, filename_prefix)

Expand Down Expand Up @@ -717,9 +718,10 @@ def download_file(url, dstpath, download_even_if_exists=False, filename_prefix='
print(']')
sys.stdout.flush()
except Exception as e:
errlog("Error: Downloading URL '" + url + "': " + str(e))
if "SSL: CERTIFICATE_VERIFY_FAILED" in str(e) or "urlopen error unknown url type: https" in str(e):
errlog("Warning: Possibly SSL/TLS issue. Update or install Python SSL root certificates (2048-bit or greater) supplied in Python folder or https://pypi.org/project/certifi/ and try again.")
if not silent:
errlog("Error: Downloading URL '" + url + "': " + str(e))
if "SSL: CERTIFICATE_VERIFY_FAILED" in str(e) or "urlopen error unknown url type: https" in str(e):
errlog("Warning: Possibly SSL/TLS issue. Update or install Python SSL root certificates (2048-bit or greater) supplied in Python folder or https://pypi.org/project/certifi/ and try again.")
rmfile(file_name)
return None
except KeyboardInterrupt:
Expand Down Expand Up @@ -1407,18 +1409,36 @@ def download_and_extract(archive, dest_dir, filename_prefix='', clobber=True):
debug_print('download_and_extract(archive=' + archive + ', dest_dir=' + dest_dir + ')')

url = urljoin(emsdk_packages_url, archive)
download_target = get_download_target(url, download_dir, filename_prefix)

received_download_target = download_file(url, download_dir, not KEEP_DOWNLOADS, filename_prefix)
if not received_download_target:
def try_download(url, silent=False):
return download_file(url, download_dir, not KEEP_DOWNLOADS,
filename_prefix, silent=silent)

# Special hack for the wasm-binaries we transitioned from `.bzip2` to
# `.xz`, but we can't tell from the version/url which one to use, so
# try one and then fall back to the other.
success = False
if 'wasm-binaries' in archive and os.path.splitext(archive)[1] == '.xz':
success = try_download(url, silent=True)
if not success:
alt_url = url.replace('.tar.xz', '.tbz2')
success = try_download(alt_url, silent=True)
if success:
url = alt_url

if not success:
success = try_download(url)

if not success:
return False
assert received_download_target == download_target

# Remove the old directory, since we have some SDKs that install into the
# same directory. If we didn't do this contents of the previous install
# could remain.
if clobber:
remove_tree(dest_dir)

download_target = get_download_target(url, download_dir, filename_prefix)
if archive.endswith('.zip'):
return unzip(download_target, dest_dir)
else:
Expand Down
8 changes: 4 additions & 4 deletions emsdk_manifest.json
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@
"version": "%releases-tag%",
"bitness": 64,
"arch": "x86_64",
"linux_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/linux/%releases-tag%/wasm-binaries.tbz2",
"macos_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/mac/%releases-tag%/wasm-binaries.tbz2",
"linux_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/linux/%releases-tag%/wasm-binaries.tar.xz",
"macos_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/mac/%releases-tag%/wasm-binaries.tar.xz",
"windows_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/win/%releases-tag%/wasm-binaries.zip",
"download_prefix": "%releases-tag%-",
"install_path": "upstream",
Expand All @@ -48,8 +48,8 @@
"version": "%releases-tag%",
"bitness": 64,
"arch": "arm64",
"macos_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/mac/%releases-tag%/wasm-binaries-arm64.tbz2",
"linux_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/linux/%releases-tag%/wasm-binaries-arm64.tbz2",
"macos_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/mac/%releases-tag%/wasm-binaries-arm64.tar.xz",
"linux_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/linux/%releases-tag%/wasm-binaries-arm64.tar.xz",
"download_prefix": "%releases-tag%-",
"install_path": "upstream",
"activated_path": "%installation_dir%/emscripten",
Expand Down

0 comments on commit 6d11425

Please sign in to comment.