From 15c16c73e434035603379abbd1c4a63be857afb9 Mon Sep 17 00:00:00 2001 From: partialord Date: Tue, 3 Sep 2024 01:41:32 +0300 Subject: [PATCH 1/2] Sat::common() fast path --- crates/ordinals/src/sat.rs | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/crates/ordinals/src/sat.rs b/crates/ordinals/src/sat.rs index 189bce8561..cba16e01d8 100644 --- a/crates/ordinals/src/sat.rs +++ b/crates/ordinals/src/sat.rs @@ -57,10 +57,16 @@ impl Sat { self.into() } - /// `Sat::rarity` is expensive and is called frequently when indexing. - /// Sat::is_common only checks if self is `Rarity::Common` but is - /// much faster. + /// Is this sat common or not? Much faster than `Sat::rarity()`. pub fn common(self) -> bool { + // The block rewards for epochs 0 through 9 are all multiples + // of 9765625 (the epoch 9 reward), so any sat from epoch 9 or + // earlier that isn't divisible by 9765625 is definitely common. + if self < Epoch(10).starting_sat() && self.0 % Epoch(9).subsidy() != 0 { + return true; + } + + // Fall back to the full calculation. let epoch = self.epoch(); (self.0 - epoch.starting_sat().0) % epoch.subsidy() != 0 } @@ -754,6 +760,16 @@ mod tests { case(2067187500000000 + 1); } + #[test] + fn common_fast_path() { + // Exhaustively test the Sat::common() fast path on every + // uncommon sat. + for height in 0..Epoch::FIRST_POST_SUBSIDY.starting_height().0 { + let height = Height(height); + assert!(!Sat::common(height.starting_sat())); + } + } + #[test] fn coin() { assert!(Sat(0).coin()); From 7e8740432d03c538a78ebe2e24ff7c942d9efbe7 Mon Sep 17 00:00:00 2001 From: partialord Date: Tue, 3 Sep 2024 13:37:29 +0300 Subject: [PATCH 2/2] keep sat ranges in low-level format --- src/index/updater.rs | 147 +++++++++++++---------- src/index/updater/inscription_updater.rs | 15 +-- src/lib.rs | 2 +- 3 files changed, 94 insertions(+), 70 deletions(-) diff --git a/src/index/updater.rs b/src/index/updater.rs index 457b5fb046..f1f62cc557 100644 --- a/src/index/updater.rs +++ b/src/index/updater.rs @@ -519,13 +519,14 @@ impl<'index> Updater<'index> { unbound_inscriptions, }; - let mut coinbase_inputs = VecDeque::new(); + let mut coinbase_inputs = Vec::new(); + let mut lost_sat_ranges = Vec::new(); if self.index.index_sats { let h = Height(self.height); if h.subsidy() > 0 { let start = h.starting_sat(); - coinbase_inputs.push_front((start.n(), (start + h.subsidy()).n())); + coinbase_inputs.extend(SatRange::store((start.n(), (start + h.subsidy()).n()))); self.sat_ranges_since_flush += 1; } } @@ -593,74 +594,36 @@ impl<'index> Updater<'index> { .map(|_| UtxoEntryBuf::new()) .collect::>(); - let mut orig_input_sat_ranges = None; + let input_sat_ranges; if self.index.index_sats { - let mut input_sat_ranges; + let leftover_sat_ranges; if tx_offset == 0 { - // We use mem::take() because the borrow checker isn't smart enough - // to realize that coinbase_inputs won't be used again. - input_sat_ranges = mem::take(&mut coinbase_inputs); + input_sat_ranges = Some(vec![coinbase_inputs.as_slice()]); + leftover_sat_ranges = &mut lost_sat_ranges; } else { - input_sat_ranges = VecDeque::new(); - - for input_utxo_entry in &input_utxo_entries { - for chunk in input_utxo_entry.sat_ranges().chunks_exact(11) { - input_sat_ranges.push_back(SatRange::load(chunk.try_into().unwrap())); - } - } + input_sat_ranges = Some( + input_utxo_entries + .iter() + .map(|entry| entry.sat_ranges()) + .collect(), + ); + leftover_sat_ranges = &mut coinbase_inputs; } - orig_input_sat_ranges = Some(input_sat_ranges.clone()); - self.index_transaction_sats( tx, *txid, &mut sat_to_satpoint, &mut output_utxo_entries, - &mut input_sat_ranges, + input_sat_ranges.as_ref().unwrap(), + leftover_sat_ranges, sat_ranges_written, outputs_in_block, )?; - - if tx_offset == 0 { - if !input_sat_ranges.is_empty() { - // Note that the lost-sats outpoint is special, because (unlike real - // outputs) it gets written to more than once. commit() will merge - // our new entry with any existing one. - let utxo_entry = utxo_cache - .entry(OutPoint::null()) - .or_insert(UtxoEntryBuf::empty(self.index)); - - let mut lost_sat_ranges = Vec::new(); - for (start, end) in input_sat_ranges { - if !Sat(start).common() { - sat_to_satpoint.insert( - &start, - &SatPoint { - outpoint: OutPoint::null(), - offset: lost_sats, - } - .store(), - )?; - } - - lost_sat_ranges.extend_from_slice(&(start, end).store()); - lost_sats += end - start; - } - - let mut new_utxo_entry = UtxoEntryBuf::new(); - new_utxo_entry.push_sat_ranges(&lost_sat_ranges, self.index); - if self.index.index_addresses { - new_utxo_entry.push_script_pubkey(&[], self.index); - } - - *utxo_entry = UtxoEntryBuf::merged(utxo_entry, &new_utxo_entry, self.index); - } - } else { - coinbase_inputs.extend(input_sat_ranges); - } } else { + input_sat_ranges = None; + for (vout, txout) in tx.output.iter().enumerate() { output_utxo_entries[vout].push_value(txout.value, self.index); } @@ -678,7 +641,7 @@ impl<'index> Updater<'index> { &mut output_utxo_entries, utxo_cache, self.index, - orig_input_sat_ranges.as_ref(), + input_sat_ranges.as_ref(), )?; } @@ -693,6 +656,39 @@ impl<'index> Updater<'index> { .insert(&self.height, inscription_updater.next_sequence_number)?; } + if !lost_sat_ranges.is_empty() { + // Note that the lost-sats outpoint is special, because (unlike real + // outputs) it gets written to more than once. commit() will merge + // our new entry with any existing one. + let utxo_entry = utxo_cache + .entry(OutPoint::null()) + .or_insert(UtxoEntryBuf::empty(self.index)); + + for chunk in lost_sat_ranges.chunks_exact(11) { + let (start, end) = SatRange::load(chunk.try_into().unwrap()); + if !Sat(start).common() { + sat_to_satpoint.insert( + &start, + &SatPoint { + outpoint: OutPoint::null(), + offset: lost_sats, + } + .store(), + )?; + } + + lost_sats += end - start; + } + + let mut new_utxo_entry = UtxoEntryBuf::new(); + new_utxo_entry.push_sat_ranges(&lost_sat_ranges, self.index); + if self.index.index_addresses { + new_utxo_entry.push_script_pubkey(&[], self.index); + } + + *utxo_entry = UtxoEntryBuf::merged(utxo_entry, &new_utxo_entry, self.index); + } + statistic_to_count.insert( &Statistic::LostSats.key(), &if self.index.index_sats { @@ -736,22 +732,43 @@ impl<'index> Updater<'index> { txid: Txid, sat_to_satpoint: &mut Table, output_utxo_entries: &mut [UtxoEntryBuf], - input_sat_ranges: &mut VecDeque<(u64, u64)>, + input_sat_ranges: &[&[u8]], + leftover_sat_ranges: &mut Vec, sat_ranges_written: &mut u64, outputs_traversed: &mut u64, ) -> Result { + let mut pending_input_sat_range = None; + let mut input_sat_ranges_iter = input_sat_ranges + .iter() + .flat_map(|slice| slice.chunks_exact(11)); + + // Preallocate our temporary array, sized to hold the combined + // sat ranges from our inputs. We'll never need more than that + // for a single output, even if we end up splitting some ranges. + let mut sats = Vec::with_capacity( + input_sat_ranges + .iter() + .map(|slice| slice.len()) + .sum::(), + ); + for (vout, output) in tx.output.iter().enumerate() { let outpoint = OutPoint { vout: vout.try_into().unwrap(), txid, }; - let mut sats = Vec::new(); let mut remaining = output.value; while remaining > 0 { - let range = input_sat_ranges - .pop_front() - .ok_or_else(|| anyhow!("insufficient inputs for transaction outputs"))?; + let range = pending_input_sat_range.take().unwrap_or_else(|| { + SatRange::load( + input_sat_ranges_iter + .next() + .expect("insufficient inputs for transaction outputs") + .try_into() + .unwrap(), + ) + }); if !Sat(range.0).common() { sat_to_satpoint.insert( @@ -769,7 +786,7 @@ impl<'index> Updater<'index> { let assigned = if count > remaining { self.sat_ranges_since_flush += 1; let middle = range.0 + remaining; - input_sat_ranges.push_front((middle, range.1)); + pending_input_sat_range = Some((middle, range.1)); (range.0, middle) } else { range @@ -785,7 +802,13 @@ impl<'index> Updater<'index> { *outputs_traversed += 1; output_utxo_entries[vout].push_sat_ranges(&sats, self.index); + sats.clear(); + } + + if let Some(range) = pending_input_sat_range { + leftover_sat_ranges.extend(&range.store()); } + leftover_sat_ranges.extend(input_sat_ranges_iter.flatten()); Ok(()) } diff --git a/src/index/updater/inscription_updater.rs b/src/index/updater/inscription_updater.rs index feaabbd619..59e4738889 100644 --- a/src/index/updater/inscription_updater.rs +++ b/src/index/updater/inscription_updater.rs @@ -67,7 +67,7 @@ impl<'a, 'tx> InscriptionUpdater<'a, 'tx> { output_utxo_entries: &mut [UtxoEntryBuf], utxo_cache: &mut HashMap, index: &Index, - input_sat_ranges: Option<&VecDeque<(u64, u64)>>, + input_sat_ranges: Option<&Vec<&[u8]>>, ) -> Result { let mut floating_inscriptions = Vec::new(); let mut id_counter = 0; @@ -340,14 +340,15 @@ impl<'a, 'tx> InscriptionUpdater<'a, 'tx> { } } - fn calculate_sat( - input_sat_ranges: Option<&VecDeque<(u64, u64)>>, - input_offset: u64, - ) -> Option { + fn calculate_sat(input_sat_ranges: Option<&Vec<&[u8]>>, input_offset: u64) -> Option { let input_sat_ranges = input_sat_ranges?; let mut offset = 0; - for (start, end) in input_sat_ranges { + for chunk in input_sat_ranges + .iter() + .flat_map(|slice| slice.chunks_exact(11)) + { + let (start, end) = SatRange::load(chunk.try_into().unwrap()); let size = end - start; if offset + size > input_offset { let n = start + input_offset - offset; @@ -361,7 +362,7 @@ impl<'a, 'tx> InscriptionUpdater<'a, 'tx> { fn update_inscription_location( &mut self, - input_sat_ranges: Option<&VecDeque<(u64, u64)>>, + input_sat_ranges: Option<&Vec<&[u8]>>, flotsam: Flotsam, new_satpoint: SatPoint, op_return: bool, diff --git a/src/lib.rs b/src/lib.rs index 00e965c600..5eb960a7ae 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -63,7 +63,7 @@ use { std::{ backtrace::BacktraceStatus, cmp, - collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque}, + collections::{BTreeMap, BTreeSet, HashMap, HashSet}, env, ffi::OsString, fmt::{self, Display, Formatter},