ethash: implement Progpow (#9762)

* ethash: initial implementation of progpow

* progpow: use wrapping arithmetic

* progpow: cleanup comments

* progpow: fix keccak_f800

* progpow: reorder definitions

* progpow: general fixing

* progpow: add basic tests from geth

* progpow: generate c_dag and add test

* progpow: fix progpow_init and progpow_loop

* progpow: fix and add new test

* progpow: tabify

* progpow: add shared testvectors from geth and aleth

* progpow: add benchmarks

* progpow: don't read bytes from dag

* ethash: use criterion for progpow benchmarks

* progpow: dont borrow hash on fnv1a_hash

* progpow: don't borrow operand on progpow merge

* progpow: hardcode dag lookup function

we only support light verification anyway

* progpow: read double words directly from the dag

* progpow: inline some small functions

* progpow: remove some bounds checking from the main loop

* progpow: remove unreachable match cases

* progpow: remove bounds check in keccak_f800_round

* progpow: fix ptr::swap

* progpow: force loop unroll in keccak_f800_round

* progpow: remove unnecessary branching in progpow_loop

* progpow: force loop unroll in fill_mix

* progpow: silence unused warning

* progpow: dont run last keccak_f800_round out of the loop

rustc generates the same assembly, it unrolls the loop

* progpow: fix output of keccak_f800_short

* ethcore: support progpow in ethash engine

* ethash: fix typo

* ethcore, ethash: fix tests

* json: fix ethash spec tests

* ethash: update quick_get_difficulty for progpow

* ethash: drop light cache on progpow transition block

* ethash: fix quick_get_difficulty tests

* progpow: update to spec v0.9.0

* progpow: update to spec v0.9.1

* progpow: update to spec v0.9.2

* ethash: rename progpow benchmarks

* fix Cargo.lock bad merge

* ethash: only export modules for benchmarks

* ethash: progpow: remove unsafe unchecked indexing

* ethash: create enum for pow algorithm

* ethash: box the progpow cdag

* ethash: skip slow progpow test vectors on ci

* ethash: don't skip progpow test vectors

they don't take too long when running in release mode which is the case
for CI.

* ethash: progpow: update copyright date

Co-Authored-By: andresilva <andre.beat@gmail.com>

* ethcore: remove verification of ci-skip-tests on non-test builds

Cargo.toml

@@ -91,7 +91,7 @@ default = ["accounts"]
 accounts = ["ethcore-accounts", "parity-rpc/accounts"]
 miner-debug = ["ethcore/miner-debug"]
 json-tests = ["ethcore/json-tests"]
-ci-skip-issue = ["ethcore/ci-skip-issue"]
+ci-skip-tests = ["ethcore/ci-skip-tests"]
 test-heavy = ["ethcore/test-heavy"]
 evm-debug = ["ethcore/evm-debug"]
 evm-debug-tests = ["ethcore/evm-debug-tests"]

ethash/Cargo.toml

@@ -14,9 +14,19 @@ parking_lot = "0.7"
 primal = "0.2.3"
 
 [dev-dependencies]
-tempdir = "0.3"
 criterion = "0.2"
+rustc-hex = "1.0"
+serde_json = "1.0"
+tempdir = "0.3"
+
+[features]
+default = []
+bench = []
 
 [[bench]]
 name = "basic"
 harness = false
+
+[[bench]]
+name = "progpow"
+harness = false

ethash/benches/basic.rs

@@ -40,38 +40,38 @@ criterion_main!(basic);
 fn bench_light_compute_memmap(b: &mut Criterion) {
 	use std::env;
 
-	let builder = NodeCacheBuilder::new(OptimizeFor::Memory);
+	let builder = NodeCacheBuilder::new(OptimizeFor::Memory, u64::max_value());
 	let light = builder.light(&env::temp_dir(), 486382);
 
-	b.bench_function("bench_light_compute_memmap", move |b| b.iter(|| light.compute(&HASH, NONCE)));
+	b.bench_function("bench_light_compute_memmap", move |b| b.iter(|| light.compute(&HASH, NONCE, u64::max_value())));
 }
 
 fn bench_light_compute_memory(b: &mut Criterion) {
 	use std::env;
 
-	let builder = NodeCacheBuilder::new(OptimizeFor::Cpu);
+	let builder = NodeCacheBuilder::new(OptimizeFor::Cpu, u64::max_value());
 	let light = builder.light(&env::temp_dir(), 486382);
 
-	b.bench_function("bench_light_compute_memmap", move |b| b.iter(|| light.compute(&HASH, NONCE)));
+	b.bench_function("bench_light_compute_memmap", move |b| b.iter(|| light.compute(&HASH, NONCE, u64::max_value())));
 }
 
 fn bench_light_new_round_trip_memmap(b: &mut Criterion) {
 	use std::env;
 
 	b.bench_function("bench_light_compute_memmap", move |b| b.iter(|| {
-		let builder = NodeCacheBuilder::new(OptimizeFor::Memory);
+		let builder = NodeCacheBuilder::new(OptimizeFor::Memory, u64::max_value());
 		let light = builder.light(&env::temp_dir(), 486382);
-		light.compute(&HASH, NONCE);
+		light.compute(&HASH, NONCE, u64::max_value());
 	}));
 }
 
 fn bench_light_new_round_trip_memory(b: &mut Criterion) {
 	use std::env;
 
 	b.bench_function("bench_light_compute_memmap", move |b| b.iter(|| {
-		let builder = NodeCacheBuilder::new(OptimizeFor::Cpu);
+		let builder = NodeCacheBuilder::new(OptimizeFor::Cpu, u64::max_value());
 		let light = builder.light(&env::temp_dir(), 486382);
-		light.compute(&HASH, NONCE);
+		light.compute(&HASH, NONCE, u64::max_value());
 	}));
 }
 
@@ -81,15 +81,15 @@ fn bench_light_from_file_round_trip_memory(b: &mut Criterion) {
 	let dir = env::temp_dir();
 	let height = 486382;
 	{
-		let builder = NodeCacheBuilder::new(OptimizeFor::Cpu);
+		let builder = NodeCacheBuilder::new(OptimizeFor::Cpu, u64::max_value());
 		let mut dummy = builder.light(&dir, height);
 		dummy.to_file().unwrap();
 	}
 
 	b.bench_function("bench_light_compute_memmap", move |b| b.iter(|| {
-		let builder = NodeCacheBuilder::new(OptimizeFor::Cpu);
+		let builder = NodeCacheBuilder::new(OptimizeFor::Cpu, u64::max_value());
 		let light = builder.light_from_file(&dir, 486382).unwrap();
-		light.compute(&HASH, NONCE);
+		light.compute(&HASH, NONCE, u64::max_value());
 	}));
 }
 
@@ -100,14 +100,14 @@ fn bench_light_from_file_round_trip_memmap(b: &mut Criterion) {
 	let height = 486382;
 
 	{
-		let builder = NodeCacheBuilder::new(OptimizeFor::Memory);
+		let builder = NodeCacheBuilder::new(OptimizeFor::Memory, u64::max_value());
 		let mut dummy = builder.light(&dir, height);
 		dummy.to_file().unwrap();
 	}
 
 	b.bench_function("bench_light_compute_memmap", move |b| b.iter(|| {
-		let builder = NodeCacheBuilder::new(OptimizeFor::Memory);
+		let builder = NodeCacheBuilder::new(OptimizeFor::Memory, u64::max_value());
 		let light = builder.light_from_file(&dir, 486382).unwrap();
-		light.compute(&HASH, NONCE);
+		light.compute(&HASH, NONCE, u64::max_value());
 	}));
 }

ethash/benches/progpow.rs

@@ -0,0 +1,86 @@
+#[macro_use]
+extern crate criterion;
+extern crate ethash;
+extern crate rustc_hex;
+extern crate tempdir;
+
+use criterion::Criterion;
+use ethash::progpow;
+
+use tempdir::TempDir;
+use rustc_hex::FromHex;
+use ethash::{NodeCacheBuilder, OptimizeFor};
+use ethash::compute::light_compute;
+
+fn bench_hashimoto_light(c: &mut Criterion) {
+	let builder = NodeCacheBuilder::new(OptimizeFor::Memory, u64::max_value());
+	let tempdir = TempDir::new("").unwrap();
+	let light = builder.light(&tempdir.path(), 1);
+	let h = FromHex::from_hex("c9149cc0386e689d789a1c2f3d5d169a61a6218ed30e74414dc736e442ef3d1f").unwrap();
+	let mut hash = [0; 32];
+	hash.copy_from_slice(&h);
+
+	c.bench_function("hashimoto_light", move |b| {
+		b.iter(|| light_compute(&light, &hash, 0))
+	});
+}
+
+fn bench_progpow_light(c: &mut Criterion) {
+	let builder = NodeCacheBuilder::new(OptimizeFor::Memory, u64::max_value());
+	let tempdir = TempDir::new("").unwrap();
+	let cache = builder.new_cache(tempdir.into_path(), 0);
+
+	let h = FromHex::from_hex("c9149cc0386e689d789a1c2f3d5d169a61a6218ed30e74414dc736e442ef3d1f").unwrap();
+	let mut hash = [0; 32];
+	hash.copy_from_slice(&h);
+
+	c.bench_function("progpow_light", move |b| {
+		b.iter(|| {
+			let c_dag = progpow::generate_cdag(cache.as_ref());
+			progpow::progpow(
+				hash,
+				0,
+				0,
+				cache.as_ref(),
+				&c_dag,
+			);
+		})
+	});
+}
+
+fn bench_progpow_optimal_light(c: &mut Criterion) {
+	let builder = NodeCacheBuilder::new(OptimizeFor::Memory, u64::max_value());
+	let tempdir = TempDir::new("").unwrap();
+	let cache = builder.new_cache(tempdir.into_path(), 0);
+	let c_dag = progpow::generate_cdag(cache.as_ref());
+
+	let h = FromHex::from_hex("c9149cc0386e689d789a1c2f3d5d169a61a6218ed30e74414dc736e442ef3d1f").unwrap();
+	let mut hash = [0; 32];
+	hash.copy_from_slice(&h);
+
+	c.bench_function("progpow_optimal_light", move |b| {
+		b.iter(|| {
+			progpow::progpow(
+				hash,
+				0,
+				0,
+				cache.as_ref(),
+				&c_dag,
+			);
+		})
+	});
+}
+
+fn bench_keccak_f800_long(c: &mut Criterion) {
+	c.bench_function("keccak_f800_long(0, 0, 0)", |b| {
+		b.iter(|| progpow::keccak_f800_long([0; 32], 0, [0; 8]))
+	});
+}
+
+criterion_group!(benches,
+	bench_hashimoto_light,
+	bench_progpow_light,
+	bench_progpow_optimal_light,
+	bench_keccak_f800_long,
+);
+criterion_main!(benches);

ethash/res/progpow_testvectors.json

@@ -0,0 +1,86 @@
+[
+  [
+    0,
+    "0000000000000000000000000000000000000000000000000000000000000000",
+    "0000000000000000",
+    "faeb1be51075b03a4ff44b335067951ead07a3b078539ace76fd56fc410557a3",
+    "63155f732f2bf556967f906155b510c917e48e99685ead76ea83f4eca03ab12b"
+  ],
+  [
+    49,
+    "63155f732f2bf556967f906155b510c917e48e99685ead76ea83f4eca03ab12b",
+    "0000000006ff2c47",
+    "c789c1180f890ec555ff42042913465481e8e6bc512cb981e1c1108dc3f2227d",
+    "9e7248f20914913a73d80a70174c331b1d34f260535ac3631d770e656b5dd922"
+  ],
+  [
+    50,
+    "9e7248f20914913a73d80a70174c331b1d34f260535ac3631d770e656b5dd922",
+    "00000000076e482e",
+    "c7340542c2a06b3a7dc7222635f7cd402abf8b528ae971ddac6bbe2b0c7cb518",
+    "de37e1824c86d35d154cf65a88de6d9286aec4f7f10c3fc9f0fa1bcc2687188d"
+  ],
+  [
+    99,
+    "de37e1824c86d35d154cf65a88de6d9286aec4f7f10c3fc9f0fa1bcc2687188d",
+    "000000003917afab",
+    "f5e60b2c5bfddd136167a30cbc3c8dbdbd15a512257dee7964e0bc6daa9f8ba7",
+    "ac7b55e801511b77e11d52e9599206101550144525b5679f2dab19386f23dcce"
+  ],
+  [
+    29950,
+    "ac7b55e801511b77e11d52e9599206101550144525b5679f2dab19386f23dcce",
+    "005d409dbc23a62a",
+    "07393d15805eb08ee6fc6cb3ad4ad1010533bd0ff92d6006850246829f18fd6e",
+    "e43d7e0bdc8a4a3f6e291a5ed790b9fa1a0948a2b9e33c844888690847de19f5"
+  ],
+  [
+    29999,
+    "e43d7e0bdc8a4a3f6e291a5ed790b9fa1a0948a2b9e33c844888690847de19f5",
+    "005db5fa4c2a3d03",
+    "7551bddf977491da2f6cfc1679299544b23483e8f8ee0931c4c16a796558a0b8",
+    "d34519f72c97cae8892c277776259db3320820cb5279a299d0ef1e155e5c6454"
+  ],
+  [
+    30000,
+    "d34519f72c97cae8892c277776259db3320820cb5279a299d0ef1e155e5c6454",
+    "005db8607994ff30",
+    "f1c2c7c32266af9635462e6ce1c98ebe4e7e3ecab7a38aaabfbf2e731e0fbff4",
+    "8b6ce5da0b06d18db7bd8492d9e5717f8b53e7e098d9fef7886d58a6e913ef64"
+  ],
+  [
+    30049,
+    "8b6ce5da0b06d18db7bd8492d9e5717f8b53e7e098d9fef7886d58a6e913ef64",
+    "005e2e215a8ca2e7",
+    "57fe6a9fbf920b4e91deeb66cb0efa971e08229d1a160330e08da54af0689add",
+    "c2c46173481b9ced61123d2e293b42ede5a1b323210eb2a684df0874ffe09047"
+  ],
+  [
+    30050,
+    "c2c46173481b9ced61123d2e293b42ede5a1b323210eb2a684df0874ffe09047",
+    "005e30899481055e",
+    "ba30c61cc5a2c74a5ecaf505965140a08f24a296d687e78720f0b48baf712f2d",
+    "ea42197eb2ba79c63cb5e655b8b1f612c5f08aae1a49ff236795a3516d87bc71"
+  ],
+  [
+    30099,
+    "ea42197eb2ba79c63cb5e655b8b1f612c5f08aae1a49ff236795a3516d87bc71",
+    "005ea6aef136f88b",
+    "cfd5e46048cd133d40f261fe8704e51d3f497fc14203ac6a9ef6a0841780b1cd",
+    "49e15ba4bf501ce8fe8876101c808e24c69a859be15de554bf85dbc095491bd6"
+  ],
+  [
+    59950,
+    "49e15ba4bf501ce8fe8876101c808e24c69a859be15de554bf85dbc095491bd6",
+    "02ebe0503bd7b1da",
+    "21511fbaa31fb9f5fc4998a754e97b3083a866f4de86fa7500a633346f56d773",
+    "f5c50ba5c0d6210ddb16250ec3efda178de857b2b1703d8d5403bd0f848e19cf"
+  ],
+  [
+    59999,
+    "f5c50ba5c0d6210ddb16250ec3efda178de857b2b1703d8d5403bd0f848e19cf",
+    "02edb6275bd221e3",
+    "653eda37d337e39d311d22be9bbd3458d3abee4e643bee4a7280a6d08106ef98",
+    "341562d10d4afb706ec2c8d5537cb0c810de02b4ebb0a0eea5ae335af6fb2e88"
+  ]
+]

ethash/src/cache.rs

@@ -69,6 +69,7 @@ pub struct NodeCacheBuilder {
 	// TODO: Remove this locking and just use an `Rc`?
 	seedhash: Arc<Mutex<SeedHashCompute>>,
 	optimize_for: OptimizeFor,
+	progpow_transition: u64,
 }
 
 // TODO: Abstract the "optimize for" logic
@@ -82,17 +83,18 @@ pub struct NodeCache {
 
 impl NodeCacheBuilder {
 	pub fn light(&self, cache_dir: &Path, block_number: u64) -> Light {
-		Light::new_with_builder(self, cache_dir, block_number)
+		Light::new_with_builder(self, cache_dir, block_number, self.progpow_transition)
 	}
 
 	pub fn light_from_file(&self, cache_dir: &Path, block_number: u64) -> io::Result<Light> {
-		Light::from_file_with_builder(self, cache_dir, block_number)
+		Light::from_file_with_builder(self, cache_dir, block_number, self.progpow_transition)
 	}
 
-	pub fn new<T: Into<Option<OptimizeFor>>>(optimize_for: T) -> Self {
+	pub fn new<T: Into<Option<OptimizeFor>>>(optimize_for: T, progpow_transition: u64) -> Self {
 		NodeCacheBuilder {
 			seedhash: Arc::new(Mutex::new(SeedHashCompute::default())),
 			optimize_for: optimize_for.into().unwrap_or_default(),
+			progpow_transition
 		}
 	}