feat: move concrete parameters from manta-sdk to manta-parameters

CHANGELOG.md

@@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased]
 ### Added
+- [\#102](https://github.com/Manta-Network/manta-rs/pull/102) Add concrete parameters to `manta-parameters`
 
 ### Changed
 

Cargo.toml

@@ -4,6 +4,7 @@ members = [
     "manta-accounting",
     "manta-benchmark",
     "manta-crypto",
+    "manta-parameters",
     "manta-pay",
     "manta-util",
 ]

manta-benchmark/Cargo.toml

@@ -40,9 +40,9 @@ instant = { version = "0.1.12", features = [ "wasm-bindgen" ] }
 manta-accounting = { path = "../manta-accounting", default-features = false, features = ["test"] }
 manta-crypto = { path = "../manta-crypto", default-features = false, features = ["getrandom", "test"] }
 manta-pay = { path = "../manta-pay", default-features = false, features = ["groth16", "test"] }
-wasm-bindgen = { version = "0.2.28", default-features = false }
+wasm-bindgen = { version = "0.2.81", default-features = false }
 wasm-bindgen-test = { version = "0.3.30", default-features = false }
-web-sys = { version = "0.3.57", default-features = false, features = ["console"] }
+web-sys = { version = "0.3.58", default-features = false, features = ["console"] }
 
 [dev-dependencies]
 criterion = { version = "0.3.4", default-features = false }

manta-parameters/.gitattributes

@@ -0,0 +1 @@
+*.lfs filter=lfs diff=lfs merge=lfs -text

manta-parameters/Cargo.toml

@@ -0,0 +1,50 @@
+[package]
+name = "manta-parameters"
+version = "0.5.0"
+edition = "2021"
+authors = ["Manta Network <contact@manta.network>"]
+readme = "README.md"
+license-file = "LICENSE"
+repository = "https://github.com/Manta-Network/manta-rs"
+homepage = "https://github.com/Manta-Network"
+documentation = "https://github.com/Manta-Network/manta-rs"
+categories = [""]
+keywords = [""]
+description = "Concrete Parameters for Manta Protocols."
+publish = false
+
+[package.metadata.docs.rs]
+# To build locally:
+# RUSTDOCFLAGS="--cfg doc_cfg" cargo +nightly doc --all-features --open
+all-features = true
+rustdoc-args = ["--cfg", "doc_cfg"]
+
+[badges]
+is-it-maintained-issue-resolution = { repository = "Manta-Network/manta-rs" }
+is-it-maintained-open-issues = { repository = "Manta-Network/manta-rs" }
+maintenance = { status = "actively-developed" }
+
+[features]
+# Download Data from GitHub
+download = ["anyhow", "attohttpc", "std"]
+
+# Enable Standard Library
+std = ["anyhow?/std"]
+
+[dependencies]
+anyhow = { version = "1.0.57", optional = true, default-features = false }
+attohttpc = { version = "0.19.1", optional = true }
+blake3 = { version = "1.3.1", default-features = false }
+
+[dev-dependencies]
+git2 = { version = "0.14.4", default-features = false }
+hex = { version = "0.4.3", default-features = false, features = ["std"] }
+manta-parameters = { path = ".", default-features = false, features = ["download"] }
+tempfile = { version = "3.3.0", default-features = false }
+walkdir = { version = "2.3.2", default-features = false }
+
+[build-dependencies]
+anyhow = { version = "1.0.57", default-features = false, features = ["std"] }
+blake3 = { version = "1.3.1", default-features = false, features = ["std"] }
+hex = { version = "0.4.3", default-features = false, features = ["std"] }
+walkdir = { version = "2.3.2", default-features = false }

manta-parameters/README.md

@@ -0,0 +1,40 @@
+# manta-parameters
+
+This is a data library that represents the parts of the Manta protocols which depend on real-world data or concrete public parameters. 
+
+- [`data`](data)
+- [`data.checkfile`](data.checkfile)
+- [`src`](src/lib.rs)
+- [`build.rs`](build.rs)
+
+The data library is comprised of a bunch of data files, either as raw binary data or JSON, and an accompanying Rust library which has access to these data files at compile-time. See the [`build.rs`](./build.rs) file for more on how those data files are parsed into Rust. Some data files are too large to package into a Rust library and so are left as stand-alone files. For these files, a [`BLAKE3`](https://github.com/BLAKE3-team/BLAKE3) digest is offered in the Rust library as a checksum.
+
+## Checksums
+
+For checksums we use [`BLAKE3`](https://github.com/BLAKE3-team/BLAKE3). Install `b3sum` with
+
+```sh
+cargo install b3sum
+```
+
+to compute the checksums for yourself. The checksums for the [`data`](./data/) directory are stored in [`data.checkfile`](./data.checkfile) which is created by the following command:
+
+```sh
+./generate_checkfile.sh
+```
+
+To check that the checkfile is up-to-date use the following command:
+
+```sh
+b3sum --check data.checkfile
+```
+
+## Validating the Dataset
+
+To check that the dataset in the [`data`](./data) directory matches the data exported by the `manta-parameters` crate, run 
+
+```sh
+cargo test --release -- --ignored --nocapture
+```
+
+which will download all the files on the GitHub source repository for the current branch and check that all the files match the known checksums.

manta-parameters/build.rs

@@ -0,0 +1,137 @@
+// Copyright 2019-2022 Manta Network.
+// This file is part of manta-rs.
+//
+// manta-rs is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// manta-rs is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with manta-rs.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Manta Parameters Build Script
+
+use anyhow::{anyhow, bail, ensure, Result};
+use hex::FromHex;
+use std::{
+    collections::HashMap,
+    env,
+    fs::{self, OpenOptions},
+    io::{BufRead, BufReader},
+    path::{Path, PathBuf},
+};
+
+/// Returns the parent of `path` which should exist relative to `OUT_DIR`.
+#[inline]
+fn parent(path: &Path) -> Result<&Path> {
+    path.parent()
+        .ok_or_else(|| anyhow!("The parent should be in the subtree of the `OUT_DIR` directory."))
+}
+
+/// Checksum
+type Checksum = [u8; 32];
+
+/// Checksum Map
+type ChecksumMap = HashMap<PathBuf, Checksum>;
+
+/// Parses the checkfile at `path` producing a [`ChecksumMap`] for all the files in the data
+/// directory.
+#[inline]
+fn parse_checkfile<P>(path: P) -> Result<ChecksumMap>
+where
+    P: AsRef<Path>,
+{
+    let file = OpenOptions::new().read(true).open(path)?;
+    let mut checksums = ChecksumMap::new();
+    for line in BufReader::new(file).lines() {
+        let line = line?;
+        let mut iter = line.split("  ");
+        match (iter.next(), iter.next(), iter.next()) {
+            (Some(checksum), Some(path), None) => {
+                checksums.insert(path.into(), Checksum::from_hex(checksum)?);
+            }
+            _ => bail!("Invalid checkfile line: {:?}", line),
+        }
+    }
+    Ok(checksums)
+}
+
+/// Gets the checksum from the `checksums` map for `path` returning an error if it was not found.
+#[inline]
+fn get_checksum<P>(checksums: &ChecksumMap, path: P) -> Result<Checksum>
+where
+    P: AsRef<Path>,
+{
+    let path = path.as_ref();
+    checksums
+        .get(path)
+        .ok_or_else(|| anyhow!("Unable to get checksum for path: {:?}", path))
+        .map(move |c| *c)
+}
+
+/// Writes the `checksum` to `path` returning an error if the write failed.
+#[inline]
+fn write_checksum<P>(path: P, checksum: Checksum) -> Result<()>
+where
+    P: AsRef<Path>,
+{
+    let path = path.as_ref();
+    fs::create_dir_all(parent(path)?)?;
+    Ok(fs::write(path.with_extension("checksum"), checksum)?)
+}
+
+/// Compiles a raw binary file by checking its BLAKE3 checksum with the `checksums` map and copying
+/// the data and checksum to the `out_dir`.
+#[inline]
+fn compile_dat(source: &Path, out_dir: &Path, checksums: &ChecksumMap) -> Result<()> {
+    let checksum = get_checksum(checksums, source)?;
+    let data = fs::read(source)?;
+    let found_checksum = blake3::hash(&data);
+    ensure!(
+        found_checksum == checksum,
+        "Checksum did not match for {:?}. Expected: {:?}, Found: {:?}. Data: {:?}",
+        source,
+        hex::encode(checksum),
+        found_checksum,
+        data,
+    );
+    let target = out_dir.join(source);
+    write_checksum(&target, checksum)?;
+    fs::copy(source, target)?;
+    Ok(())
+}
+
+/// Compiles a Git LFS file by writing its checksum to the `out_dir`.
+#[inline]
+fn compile_lfs(source: &Path, out_dir: &Path, checksums: &ChecksumMap) -> Result<()> {
+    write_checksum(out_dir.join(source), get_checksum(checksums, source)?)
+}
+
+/// Loads all the files from `data` into the `OUT_DIR` directory for inclusion into the library.
+#[inline]
+fn main() -> Result<()> {
+    println!("cargo:rerun-if-changed=data");
+    let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap());
+    println!("out_dir: {:?}", out_dir);
+    let checksums = parse_checkfile("data.checkfile")?;
+    for file in walkdir::WalkDir::new("data") {
+        let file = file?;
+        let path = file.path();
+        if !path.is_dir() {
+            match path.extension() {
+                Some(extension) => match extension.to_str() {
+                    Some("dat") => compile_dat(path, &out_dir, &checksums)?,
+                    Some("lfs") => compile_lfs(path, &out_dir, &checksums)?,
+                    _ => bail!("Unsupported data file extension."),
+                },
+                _ => bail!("All data files must have an extension."),
+            }
+        }
+    }
+    Ok(())
+}

manta-parameters/data.checkfile

@@ -0,0 +1,10 @@
+df87563f5a6366a341f4dd1796ecd7eec85354e0e8e0bb9e2a8cdb33be3a31e9  data/pay/testnet/parameters/note-encryption-scheme.dat
+05e6b3b6a0d6d6debe458b6f7ddaefa51a4d38608024a98fc4dedf5ee8b68e99  data/pay/testnet/parameters/utxo-accumulator-model.dat
+348d7e0072ffdb56fd19d6115c89a3c9172639c5167e8a7e905dc8a575e4f17d  data/pay/testnet/parameters/utxo-commitment-scheme.dat
+9dcf5d51f6ffbcda46eaaab64e7350095465f8cc774f8276b46889c46479a2e4  data/pay/testnet/parameters/void-number-commitment-scheme.dat
+d0dbeec5afcd85025678bf64db3a540622be09e3208ace153446639a7d5de98e  data/pay/testnet/proving/mint.lfs
+04b294703a2588e6a6a4bd98734fb18fa5f8c10b33be63bb9a73f68c05ccaf2c  data/pay/testnet/proving/private-transfer.lfs
+eb5c880bdf3c998a9a081f0259fd536e07f4bb71095bcef5664326bcb1ad6428  data/pay/testnet/proving/reclaim.lfs
+5719387f9625828f46835a8375656578e028d8fc6da6822b765f47e296c0aaac  data/pay/testnet/verifying/mint.dat
+29a8229b59490223372c1f2b918f10d806be64ac4ffa5695dbdfe97b4b52e404  data/pay/testnet/verifying/private-transfer.dat
+bbe115020d563d63d404437c38741f0d527ab0441b4aaf4de463d9e9452dee09  data/pay/testnet/verifying/reclaim.dat

manta-parameters/data/pay/testnet/parameters/note-encryption-scheme.dat

@@ -0,0 +1 @@
+.B���04����1�tLo�'5'�����ȝQ<Om

manta-parameters/generate_checkfile.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+shopt -s globstar
+b3sum data/** 2>/dev/null > data.checkfile