Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fixDAA; implement wtema #3477

Merged
merged 13 commits into from
Nov 25, 2020
2 changes: 1 addition & 1 deletion chain/src/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -368,7 +368,7 @@ impl Default for Tip {
height: 0,
last_block_h: ZERO_HASH,
prev_block_h: ZERO_HASH,
total_difficulty: Difficulty::min(),
total_difficulty: Difficulty::min_dma(),
}
}
}
Expand Down
15 changes: 9 additions & 6 deletions chain/tests/test_coinbase_maturity.rs
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ fn test_coinbase_maturity() {

let next_header_info = consensus::next_difficulty(1, chain.difficulty_iter().unwrap());
let reward = libtx::reward::output(&keychain, &builder, &key_id1, 0, false).unwrap();
let mut block = core::core::Block::new(&prev, &[], Difficulty::min(), reward).unwrap();
let mut block = core::core::Block::new(&prev, &[], Difficulty::min_dma(), reward).unwrap();
block.header.timestamp = prev.timestamp + Duration::seconds(60);
block.header.pow.secondary_scaling = next_header_info.secondary_scaling;

Expand Down Expand Up @@ -113,7 +113,7 @@ fn test_coinbase_maturity() {
let txs = &[coinbase_txn.clone()];
let fees = txs.iter().map(|tx| tx.fee()).sum();
let reward = libtx::reward::output(&keychain, &builder, &key_id3, fees, false).unwrap();
let mut block = core::core::Block::new(&prev, txs, Difficulty::min(), reward).unwrap();
let mut block = core::core::Block::new(&prev, txs, Difficulty::min_dma(), reward).unwrap();
let next_header_info = consensus::next_difficulty(1, chain.difficulty_iter().unwrap());
block.header.timestamp = prev.timestamp + Duration::seconds(60);
block.header.pow.secondary_scaling = next_header_info.secondary_scaling;
Expand Down Expand Up @@ -149,7 +149,8 @@ fn test_coinbase_maturity() {

let next_header_info = consensus::next_difficulty(1, chain.difficulty_iter().unwrap());
let reward = libtx::reward::output(&keychain, &builder, &key_id1, 0, false).unwrap();
let mut block = core::core::Block::new(&prev, &[], Difficulty::min(), reward).unwrap();
let mut block =
core::core::Block::new(&prev, &[], Difficulty::min_dma(), reward).unwrap();

block.header.timestamp = prev.timestamp + Duration::seconds(60);
block.header.pow.secondary_scaling = next_header_info.secondary_scaling;
Expand Down Expand Up @@ -195,7 +196,8 @@ fn test_coinbase_maturity() {
let txs = &[coinbase_txn.clone()];
let fees = txs.iter().map(|tx| tx.fee()).sum();
let reward = libtx::reward::output(&keychain, &builder, &key_id3, fees, false).unwrap();
let mut block = core::core::Block::new(&prev, txs, Difficulty::min(), reward).unwrap();
let mut block =
core::core::Block::new(&prev, txs, Difficulty::min_dma(), reward).unwrap();
let next_header_info = consensus::next_difficulty(1, chain.difficulty_iter().unwrap());
block.header.timestamp = prev.timestamp + Duration::seconds(60);
block.header.pow.secondary_scaling = next_header_info.secondary_scaling;
Expand Down Expand Up @@ -231,7 +233,7 @@ fn test_coinbase_maturity() {

let reward = libtx::reward::output(&keychain, &builder, &pk, 0, false).unwrap();
let mut block =
core::core::Block::new(&prev, &[], Difficulty::min(), reward).unwrap();
core::core::Block::new(&prev, &[], Difficulty::min_dma(), reward).unwrap();
let next_header_info =
consensus::next_difficulty(1, chain.difficulty_iter().unwrap());
block.header.timestamp = prev.timestamp + Duration::seconds(60);
Expand Down Expand Up @@ -262,7 +264,8 @@ fn test_coinbase_maturity() {
let fees = txs.iter().map(|tx| tx.fee()).sum();
let next_header_info = consensus::next_difficulty(1, chain.difficulty_iter().unwrap());
let reward = libtx::reward::output(&keychain, &builder, &key_id4, fees, false).unwrap();
let mut block = core::core::Block::new(&prev, txs, Difficulty::min(), reward).unwrap();
let mut block =
core::core::Block::new(&prev, txs, Difficulty::min_dma(), reward).unwrap();

block.header.timestamp = prev.timestamp + Duration::seconds(60);
block.header.pow.secondary_scaling = next_header_info.secondary_scaling;
Expand Down
13 changes: 13 additions & 0 deletions config/src/comments.rs
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,19 @@ fn comments() -> HashMap<String, String> {
.to_string(),
);

retval.insert(
"future_time_limit".to_string(),
"
#The Future Time Limit (FTL) is a limit on how far into the future,
#relative to a node's local time, the timestamp on a new block can be,
#in order for the block to be accepted.
#At Hard Fork 4, this was reduced from 12 minutes down to 5 minutes,
#so as to limit possible timestamp manipulation on the new
#wtema difficulty adjustment algorithm
"
.to_string(),
);

retval.insert(
"chain_validation_mode".to_string(),
"
Expand Down
136 changes: 91 additions & 45 deletions core/src/consensus.rs
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,11 @@ pub fn reward(fee: u64) -> u64 {
REWARD.saturating_add(fee)
}

/// an hour in seconds
pub const HOUR_SEC: u64 = 60 * 60;

/// Nominal height for standard time intervals, hour is 60 blocks
pub const HOUR_HEIGHT: u64 = 3600 / BLOCK_TIME_SEC;
pub const HOUR_HEIGHT: u64 = HOUR_SEC / BLOCK_TIME_SEC;
/// A day is 1440 blocks
pub const DAY_HEIGHT: u64 = 24 * HOUR_HEIGHT;
/// A week is 10_080 blocks
Expand All @@ -66,12 +69,6 @@ pub fn secondary_pow_ratio(height: u64) -> u64 {
90u64.saturating_sub(height / (2 * YEAR_HEIGHT / 90))
}

/// The AR scale damping factor to use. Dependent on block height
/// to account for pre HF behavior on testnet4.
fn ar_scale_damp_factor(_height: u64) -> u64 {
AR_SCALE_DAMP_FACTOR
}

/// Cuckoo-cycle proof size (cycle length)
pub const PROOFSIZE: usize = 42;

Expand Down Expand Up @@ -187,18 +184,21 @@ pub fn valid_header_version(height: u64, version: HeaderVersion) -> bool {
height < 4 * HARD_FORK_INTERVAL && version == header_version(height)
}

/// Number of blocks used to calculate difficulty adjustments
pub const DIFFICULTY_ADJUST_WINDOW: u64 = HOUR_HEIGHT;
/// Number of blocks used to calculate difficulty adjustment by Damped Moving Average
pub const DMA_WINDOW: u64 = HOUR_HEIGHT;

/// Difficulty adjustment half life is 4 hours
pub const WTEMA_HALF_LIFE: u64 = 4 * HOUR_SEC;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How was WTEMA_HALF_LIFE parameter selection determined?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it's a tradeoff between response speed and stability. the 240 blocks closely matches BTH's choice of the equivalent of 288 blocks.


/// Average time span of the difficulty adjustment window
pub const BLOCK_TIME_WINDOW: u64 = DIFFICULTY_ADJUST_WINDOW * BLOCK_TIME_SEC;
/// Average time span of the DMA difficulty adjustment window
pub const BLOCK_TIME_WINDOW: u64 = DMA_WINDOW * BLOCK_TIME_SEC;

/// Clamp factor to use for difficulty adjustment
/// Clamp factor to use for DMA difficulty adjustment
/// Limit value to within this factor of goal
pub const CLAMP_FACTOR: u64 = 2;

/// Dampening factor to use for difficulty adjustment
pub const DIFFICULTY_DAMP_FACTOR: u64 = 3;
/// Dampening factor to use for DMA difficulty adjustment
pub const DMA_DAMP_FACTOR: u64 = 3;

/// Dampening factor to use for AR scale calculation.
pub const AR_SCALE_DAMP_FACTOR: u64 = 13;
Expand All @@ -218,9 +218,16 @@ pub fn graph_weight(height: u64, edge_bits: u8) -> u64 {
(2u64 << (edge_bits - global::base_edge_bits()) as u64) * xpr_edge_bits
}

/// Minimum difficulty, enforced in diff retargetting
/// minimum solution difficulty after HardFork4 when PoW becomes primary only Cuckatoo32+
pub const C32_GRAPH_WEIGHT: u64 = (2u64 << (32 - BASE_EDGE_BITS) as u64) * 32; // 16384

/// Minimum difficulty, enforced in Damped Moving Average diff retargetting
/// avoids getting stuck when trying to increase difficulty subject to dampening
pub const MIN_DIFFICULTY: u64 = DIFFICULTY_DAMP_FACTOR;
pub const MIN_DMA_DIFFICULTY: u64 = DMA_DAMP_FACTOR;

/// Minimum difficulty, enforced in Weighted Target Exponential Moving Average diff retargetting
/// avoids getting stuck when trying to increase difficulty
pub const MIN_WTEMA_DIFFICULTY: u64 = C32_GRAPH_WEIGHT;

/// Minimum scaling factor for AR pow, enforced in diff retargetting
/// avoids getting stuck when trying to increase ar_scale subject to dampening
Expand Down Expand Up @@ -306,37 +313,43 @@ pub fn clamp(actual: u64, goal: u64, clamp_factor: u64) -> u64 {
max(goal / clamp_factor, min(actual, goal * clamp_factor))
}

/// Computes the proof-of-work difficulty that the next block should comply
/// with. Takes an iterator over past block headers information, from latest
/// Computes the proof-of-work difficulty that the next block should comply with.
/// Takes an iterator over past block headers information, from latest
/// (highest height) to oldest (lowest height).
///
/// The difficulty calculation is based on both Digishield and GravityWave
/// family of difficulty computation, coming to something very close to Zcash.
/// The reference difficulty is an average of the difficulty over a window of
/// DIFFICULTY_ADJUST_WINDOW blocks. The corresponding timespan is calculated
/// by using the difference between the median timestamps at the beginning
/// and the end of the window.
///
/// The secondary proof-of-work factor is calculated along the same lines, as
/// an adjustment on the deviation against the ideal value.
/// Uses either the old dma DAA or, starting from HF4, the new wtema DAA
pub fn next_difficulty<T>(height: u64, cursor: T) -> HeaderInfo
where
T: IntoIterator<Item = HeaderInfo>,
{
if header_version(height) < HeaderVersion(5) {
next_dma_difficulty(height, cursor)
} else {
next_wtema_difficulty(height, cursor)
}
}

/// Difficulty calculation based on a Damped Moving Average
/// of difficulty over a window of DMA_WINDOW blocks.
/// The corresponding timespan is calculated
/// by using the difference between the timestamps at the beginning
/// and the end of the window, with a damping toward the target block time.
pub fn next_dma_difficulty<T>(height: u64, cursor: T) -> HeaderInfo
where
T: IntoIterator<Item = HeaderInfo>,
{
// Create vector of difficulty data running from earliest
// to latest, and pad with simulated pre-genesis data to allow earlier
// adjustment if there isn't enough window data length will be
// DIFFICULTY_ADJUST_WINDOW + 1 (for initial block time bound)
// DMA_WINDOW + 1 (for initial block time bound)
let diff_data = global::difficulty_data_to_vector(cursor);

// First, get the ratio of secondary PoW vs primary, skipping initial header
let sec_pow_scaling = secondary_pow_scaling(height, &diff_data[1..]);

// Get the timestamp delta across the window
let ts_delta: u64 =
diff_data[DIFFICULTY_ADJUST_WINDOW as usize].timestamp - diff_data[0].timestamp;
let ts_delta: u64 = diff_data[DMA_WINDOW as usize].timestamp - diff_data[0].timestamp;

// Get the difficulty sum of the last DIFFICULTY_ADJUST_WINDOW elements
// Get the difficulty sum of the last DMA_WINDOW elements
let diff_sum: u64 = diff_data
.iter()
.skip(1)
Expand All @@ -345,37 +358,66 @@ where

// adjust time delta toward goal subject to dampening and clamping
let adj_ts = clamp(
damp(ts_delta, BLOCK_TIME_WINDOW, DIFFICULTY_DAMP_FACTOR),
damp(ts_delta, BLOCK_TIME_WINDOW, DMA_DAMP_FACTOR),
BLOCK_TIME_WINDOW,
CLAMP_FACTOR,
);
// minimum difficulty avoids getting stuck due to dampening
let difficulty = max(MIN_DIFFICULTY, diff_sum * BLOCK_TIME_SEC / adj_ts);
let difficulty = max(MIN_DMA_DIFFICULTY, diff_sum * BLOCK_TIME_SEC / adj_ts);

HeaderInfo::from_diff_scaling(Difficulty::from_num(difficulty), sec_pow_scaling)
}

/// Difficulty calculation based on a Weighted Target Exponential Moving Average
/// of difficulty, using the ratio of the last block time over the target block time.
pub fn next_wtema_difficulty<T>(_height: u64, cursor: T) -> HeaderInfo
where
T: IntoIterator<Item = HeaderInfo>,
{
let mut last_headers = cursor.into_iter();

// last two headers
let last_header = last_headers.next().unwrap();
let prev_header = last_headers.next().unwrap();

let last_block_time: u64 = last_header.timestamp - prev_header.timestamp;

let last_diff = last_header.difficulty.to_num();

// wtema difficulty update
let next_diff =
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm struggling a bit verifying this algorithm against the details in the RFC and WTEMA specification. It seems straightforward enough here, but I think I need to better wrap my head around the WTEMA_HALF_LIFE value. Maybe a line or two of documentation could be helpful here.

last_diff * WTEMA_HALF_LIFE / (WTEMA_HALF_LIFE - BLOCK_TIME_SEC + last_block_time);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Took me a bit to process so putting a note here: in terms of the RFC, WTEMA_HALF_LIFE corresponds to W * T. So in this PR we have a W of 240, meaning the difficulty can change by a factor ~e every 240 blocks (4 hours, hence the name of WTEMA_HALF_LIFE) 👍

Copy link
Contributor Author

@tromp tromp Nov 24, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

correct. the name HALF_LIFE is a bit misleading in that sense, since it corresponds not to a doubling.
but that's the terminology that jtoomim and zawy seem to use.


// minimum difficulty at graph_weight(32) ensures difficulty increase on 59s block
// since 16384 * WTEMA_HALF_LIFE / (WTEMA_HALF_LIFE - 1) > 16384
let difficulty = max(MIN_WTEMA_DIFFICULTY, next_diff);

HeaderInfo::from_diff_scaling(Difficulty::from_num(difficulty), 0) // no more secondary PoW
}

/// Count, in units of 1/100 (a percent), the number of "secondary" (AR) blocks in the provided window of blocks.
pub fn ar_count(_height: u64, diff_data: &[HeaderInfo]) -> u64 {
100 * diff_data.iter().filter(|n| n.is_secondary).count() as u64
}

/// The secondary proof-of-work factor is calculated along the same lines as in next_dma_difficulty,
/// as an adjustment on the deviation against the ideal value.
/// Factor by which the secondary proof of work difficulty will be adjusted
pub fn secondary_pow_scaling(height: u64, diff_data: &[HeaderInfo]) -> u32 {
// Get the scaling factor sum of the last DIFFICULTY_ADJUST_WINDOW elements
// Get the scaling factor sum of the last DMA_WINDOW elements
let scale_sum: u64 = diff_data.iter().map(|dd| dd.secondary_scaling as u64).sum();

// compute ideal 2nd_pow_fraction in pct and across window
let target_pct = secondary_pow_ratio(height);
let target_count = DIFFICULTY_ADJUST_WINDOW * target_pct;
let target_count = DMA_WINDOW * target_pct;

// Get the secondary count across the window, adjusting count toward goal
// subject to dampening and clamping.
let adj_count = clamp(
damp(
ar_count(height, diff_data),
target_count,
ar_scale_damp_factor(height),
AR_SCALE_DAMP_FACTOR,
),
target_count,
CLAMP_FACTOR,
Expand All @@ -396,12 +438,12 @@ mod test {

// initial weights
assert_eq!(graph_weight(1, 31), 256 * 31);
assert_eq!(graph_weight(1, 32), 512 * 32);
assert_eq!(graph_weight(1, 32), C32_GRAPH_WEIGHT);
assert_eq!(graph_weight(1, 33), 1024 * 33);

// one year in, 31 starts going down, the rest stays the same
assert_eq!(graph_weight(YEAR_HEIGHT, 31), 256 * 30);
assert_eq!(graph_weight(YEAR_HEIGHT, 32), 512 * 32);
assert_eq!(graph_weight(YEAR_HEIGHT, 32), C32_GRAPH_WEIGHT);
assert_eq!(graph_weight(YEAR_HEIGHT, 33), 1024 * 33);

// 31 loses one factor per week
Expand All @@ -411,29 +453,33 @@ mod test {

// 2 years in, 31 still at 0, 32 starts decreasing
assert_eq!(graph_weight(2 * YEAR_HEIGHT, 31), 0);
assert_eq!(graph_weight(2 * YEAR_HEIGHT, 32), 512 * 32);
assert_eq!(graph_weight(2 * YEAR_HEIGHT, 32), C32_GRAPH_WEIGHT);
assert_eq!(graph_weight(2 * YEAR_HEIGHT, 33), 1024 * 33);

// 32 phaseout on hold
assert_eq!(graph_weight(2 * YEAR_HEIGHT + WEEK_HEIGHT, 32), 512 * 32);
assert_eq!(
graph_weight(2 * YEAR_HEIGHT + WEEK_HEIGHT, 32),
C32_GRAPH_WEIGHT
);
assert_eq!(graph_weight(2 * YEAR_HEIGHT + WEEK_HEIGHT, 31), 0);
assert_eq!(
graph_weight(2 * YEAR_HEIGHT + 30 * WEEK_HEIGHT, 32),
512 * 32
C32_GRAPH_WEIGHT
);
assert_eq!(
graph_weight(2 * YEAR_HEIGHT + 31 * WEEK_HEIGHT, 32),
512 * 32
C32_GRAPH_WEIGHT
);

// 3 years in, nothing changes
assert_eq!(graph_weight(3 * YEAR_HEIGHT, 31), 0);
assert_eq!(graph_weight(3 * YEAR_HEIGHT, 32), 512 * 32);
assert_eq!(graph_weight(3 * YEAR_HEIGHT, 32), C32_GRAPH_WEIGHT);
assert_eq!(graph_weight(3 * YEAR_HEIGHT, 33), 1024 * 33);

// 4 years in, still on hold
assert_eq!(graph_weight(4 * YEAR_HEIGHT, 31), 0);
assert_eq!(graph_weight(4 * YEAR_HEIGHT, 32), 512 * 32);
assert_eq!(graph_weight(4 * YEAR_HEIGHT, 32), C32_GRAPH_WEIGHT);
assert_eq!(graph_weight(4 * YEAR_HEIGHT, 33), 1024 * 33);
assert_eq!(graph_weight(4 * YEAR_HEIGHT, 33), 1024 * 33);
}
}
12 changes: 6 additions & 6 deletions core/src/core/block.rs
Original file line number Diff line number Diff line change
Expand Up @@ -432,14 +432,14 @@ pub struct UntrustedBlockHeader(BlockHeader);
impl Readable for UntrustedBlockHeader {
fn read<R: Reader>(reader: &mut R) -> Result<UntrustedBlockHeader, ser::Error> {
let header = read_block_header(reader)?;
if header.timestamp
> Utc::now() + Duration::seconds(12 * (consensus::BLOCK_TIME_SEC as i64))
{
// refuse blocks more than 12 blocks intervals in future (as in bitcoin)
let ftl = global::get_future_time_limit();
if header.timestamp > Utc::now() + Duration::seconds(ftl as i64) {
// refuse blocks whose timestamp is too far in the future
// this future_time_limit (FTL) is specified in grin-server.toml
// TODO add warning in p2p code if local time is too different from peers
error!(
"block header {} validation error: block time is more than 12 blocks in future",
header.hash()
"block header {} validation error: block time is more than {} seconds in the future",
header.hash(), ftl
);
return Err(ser::Error::CorruptedData);
}
Expand Down
Loading