Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use zstd over gzip when not compiling for WASM #44

Merged
merged 15 commits into from
Jul 1, 2019
Prev Previous commit
Next Next commit
Remove flate path for wasm
NOTE: At least on my machine, this doesn't
compile correctly for wasm, as the zstd features don't
seem to be working correctly, eg ZSTD_LEGACY_SUPPORT=1 is passed to the
C compiler, even though that feature should
be disabled.
  • Loading branch information
Jake-Shadle committed Jun 28, 2019

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
commit a99833e890bd596f976f26eac709489d5d39f519
2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
@@ -30,7 +30,7 @@ rayon = "1.1.0"
zstd = "0.4.24+zstd.1.4.0"
Jake-Shadle marked this conversation as resolved.
Show resolved Hide resolved

[target.'cfg(target_arch = "wasm32")'.dependencies]
flate2 = { version = "1.0.8", features = ["rust_backend"], default_features = false }
zstd = { version = "0.4.24+zstd.1.4.0", default-features = false, features = ["wasm"] }

[dev-dependencies]
env_logger = "0.6.1"
68 changes: 13 additions & 55 deletions src/store/cache.rs
Original file line number Diff line number Diff line change
@@ -10,9 +10,6 @@ use serde::Serialize;

use crate::store::base::Store;

#[cfg(target_arch = "wasm32")]
const CACHE_VERSION: &[u8] = b"askalono-03";
#[cfg(not(target_arch = "wasm32"))]
const CACHE_VERSION: &[u8] = b"askalono-04";

impl Store {
@@ -23,46 +20,20 @@ impl Store {
/// the full SPDX set from disk in 200-300 ms. The cache will be
/// sanity-checked to ensure it was generated with a similar version of
/// askalono.
pub fn from_cache<R>(readable: R) -> Result<Store, Error>
pub fn from_cache<R>(mut readable: R) -> Result<Store, Error>
where
R: Read + Sized,
{
use rmp_serde::decode::from_read;
let mut header = [0u8; 11];
readable.read_exact(&mut header)?;

#[cfg(target_arch = "wasm32")]
{
let dec = flate2::read::GzDecoder::new(readable);
{
let extra = dec
.header()
.ok_or_else(|| failure::format_err!("cache gzip header invalid"))?
.extra()
.ok_or_else(|| failure::format_err!("cache gzip extra header missing"))?;
if extra != CACHE_VERSION {
failure::bail!("cache version mismatch");
}
}

Ok(from_read(dec)?)
if header != CACHE_VERSION {
failure::bail!("cache version mismatch");
}

#[cfg(not(target_arch = "wasm32"))]
{
// `readable` only has to be mut in the zstd path, so if we put it in the function
// signature the gzip path complains it doesn't need to be mut, so just move it
let mut readable = readable;

let mut header = [0u8; 11];
readable.read_exact(&mut header)?;

if header != CACHE_VERSION {
failure::bail!("cache version mismatch");
}

let dec = zstd::Decoder::new(readable)?;
let store = from_read(dec)?;
Ok(store)
}
let dec = zstd::Decoder::new(readable)?;
let store = rmp_serde::decode::from_read(dec)?;
Ok(store)
}

/// Serialize the current store.
@@ -83,25 +54,12 @@ impl Store {

info!("Pre-compressed output is {} bytes", buf.len());

#[cfg(target_arch = "wasm32")]
{
let mut gz = flate2::GzBuilder::new()
.extra(CACHE_VERSION)
.write(&mut writable, flate2::Compression::best());
copy(&mut buf.as_slice(), &mut gz)?;
writable.write_all(CACHE_VERSION)?;
let mut zenc = zstd::Encoder::new(writable, 21)?;

Ok(())
}

#[cfg(not(target_arch = "wasm32"))]
{
writable.write_all(CACHE_VERSION)?;
let mut zenc = zstd::Encoder::new(writable, 21)?;
copy(&mut buf.as_slice(), &mut zenc)?;
zenc.finish()?;

copy(&mut buf.as_slice(), &mut zenc)?;
zenc.finish()?;

Ok(())
}
Ok(())
}
}
114 changes: 21 additions & 93 deletions wasm/pkg/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion wasm/pkg/build.rs
Original file line number Diff line number Diff line change
@@ -8,7 +8,7 @@ use std::path::Path;

use askalono::Store;

const EMBEDDED_CACHE: &str = "wasm-embedded-cache.bin.gz";
const EMBEDDED_CACHE: &str = "wasm-embedded-cache.bin.zstd";

// copied over from the CLI