More efficient sat-range handling

crates/ordinals/src/sat.rs

@@ -57,10 +57,16 @@ impl Sat {
     self.into()
   }
 
-  /// `Sat::rarity` is expensive and is called frequently when indexing.
-  /// Sat::is_common only checks if self is `Rarity::Common` but is
-  /// much faster.
+  /// Is this sat common or not?  Much faster than `Sat::rarity()`.
   pub fn common(self) -> bool {
+    // The block rewards for epochs 0 through 9 are all multiples
+    // of 9765625 (the epoch 9 reward), so any sat from epoch 9 or
+    // earlier that isn't divisible by 9765625 is definitely common.
+    if self < Epoch(10).starting_sat() && self.0 % Epoch(9).subsidy() != 0 {
+      return true;
+    }
+
+    // Fall back to the full calculation.
     let epoch = self.epoch();
     (self.0 - epoch.starting_sat().0) % epoch.subsidy() != 0
   }
@@ -754,6 +760,16 @@ mod tests {
     case(2067187500000000 + 1);
   }
 
+  #[test]
+  fn common_fast_path() {
+    // Exhaustively test the Sat::common() fast path on every
+    // uncommon sat.
+    for height in 0..Epoch::FIRST_POST_SUBSIDY.starting_height().0 {
+      let height = Height(height);
+      assert!(!Sat::common(height.starting_sat()));
+    }
+  }
+
   #[test]
   fn coin() {
     assert!(Sat(0).coin());

src/index/updater.rs

@@ -519,13 +519,14 @@ impl<'index> Updater<'index> {
       unbound_inscriptions,
     };
 
-    let mut coinbase_inputs = VecDeque::new();
+    let mut coinbase_inputs = Vec::new();
+    let mut lost_sat_ranges = Vec::new();
 
     if self.index.index_sats {
       let h = Height(self.height);
       if h.subsidy() > 0 {
         let start = h.starting_sat();
-        coinbase_inputs.push_front((start.n(), (start + h.subsidy()).n()));
+        coinbase_inputs.extend(SatRange::store((start.n(), (start + h.subsidy()).n())));
         self.sat_ranges_since_flush += 1;
       }
     }
@@ -593,74 +594,36 @@ impl<'index> Updater<'index> {
         .map(|_| UtxoEntryBuf::new())
         .collect::<Vec<UtxoEntryBuf>>();
 
-      let mut orig_input_sat_ranges = None;
+      let input_sat_ranges;
       if self.index.index_sats {
-        let mut input_sat_ranges;
+        let leftover_sat_ranges;
 
         if tx_offset == 0 {
-          // We use mem::take() because the borrow checker isn't smart enough
-          // to realize that coinbase_inputs won't be used again.
-          input_sat_ranges = mem::take(&mut coinbase_inputs);
+          input_sat_ranges = Some(vec![coinbase_inputs.as_slice()]);
+          leftover_sat_ranges = &mut lost_sat_ranges;
         } else {
-          input_sat_ranges = VecDeque::new();
-
-          for input_utxo_entry in &input_utxo_entries {
-            for chunk in input_utxo_entry.sat_ranges().chunks_exact(11) {
-              input_sat_ranges.push_back(SatRange::load(chunk.try_into().unwrap()));
-            }
-          }
+          input_sat_ranges = Some(
+            input_utxo_entries
+              .iter()
+              .map(|entry| entry.sat_ranges())
+              .collect(),
+          );
+          leftover_sat_ranges = &mut coinbase_inputs;
         }
 
-        orig_input_sat_ranges = Some(input_sat_ranges.clone());
-
         self.index_transaction_sats(
           tx,
           *txid,
           &mut sat_to_satpoint,
           &mut output_utxo_entries,
-          &mut input_sat_ranges,
+          input_sat_ranges.as_ref().unwrap(),
+          leftover_sat_ranges,
           sat_ranges_written,
           outputs_in_block,
         )?;
-
-        if tx_offset == 0 {
-          if !input_sat_ranges.is_empty() {
-            // Note that the lost-sats outpoint is special, because (unlike real
-            // outputs) it gets written to more than once.  commit() will merge
-            // our new entry with any existing one.
-            let utxo_entry = utxo_cache
-              .entry(OutPoint::null())
-              .or_insert(UtxoEntryBuf::empty(self.index));
-
-            let mut lost_sat_ranges = Vec::new();
-            for (start, end) in input_sat_ranges {
-              if !Sat(start).common() {
-                sat_to_satpoint.insert(
-                  &start,
-                  &SatPoint {
-                    outpoint: OutPoint::null(),
-                    offset: lost_sats,
-                  }
-                  .store(),
-                )?;
-              }
-
-              lost_sat_ranges.extend_from_slice(&(start, end).store());
-              lost_sats += end - start;
-            }
-
-            let mut new_utxo_entry = UtxoEntryBuf::new();
-            new_utxo_entry.push_sat_ranges(&lost_sat_ranges, self.index);
-            if self.index.index_addresses {
-              new_utxo_entry.push_script_pubkey(&[], self.index);
-            }
-
-            *utxo_entry = UtxoEntryBuf::merged(utxo_entry, &new_utxo_entry, self.index);
-          }
-        } else {
-          coinbase_inputs.extend(input_sat_ranges);
-        }
       } else {
+        input_sat_ranges = None;
+
         for (vout, txout) in tx.output.iter().enumerate() {
           output_utxo_entries[vout].push_value(txout.value, self.index);
         }
@@ -678,7 +641,7 @@ impl<'index> Updater<'index> {
           &mut output_utxo_entries,
           utxo_cache,
           self.index,
-          orig_input_sat_ranges.as_ref(),
+          input_sat_ranges.as_ref(),
         )?;
       }
 
@@ -693,6 +656,39 @@ impl<'index> Updater<'index> {
         .insert(&self.height, inscription_updater.next_sequence_number)?;
     }
 
+    if !lost_sat_ranges.is_empty() {
+      // Note that the lost-sats outpoint is special, because (unlike real
+      // outputs) it gets written to more than once.  commit() will merge
+      // our new entry with any existing one.
+      let utxo_entry = utxo_cache
+        .entry(OutPoint::null())
+        .or_insert(UtxoEntryBuf::empty(self.index));
+
+      for chunk in lost_sat_ranges.chunks_exact(11) {
+        let (start, end) = SatRange::load(chunk.try_into().unwrap());
+        if !Sat(start).common() {
+          sat_to_satpoint.insert(
+            &start,
+            &SatPoint {
+              outpoint: OutPoint::null(),
+              offset: lost_sats,
+            }
+            .store(),
+          )?;
+        }
+
+        lost_sats += end - start;
+      }
+
+      let mut new_utxo_entry = UtxoEntryBuf::new();
+      new_utxo_entry.push_sat_ranges(&lost_sat_ranges, self.index);
+      if self.index.index_addresses {
+        new_utxo_entry.push_script_pubkey(&[], self.index);
+      }
+
+      *utxo_entry = UtxoEntryBuf::merged(utxo_entry, &new_utxo_entry, self.index);
+    }
+
     statistic_to_count.insert(
       &Statistic::LostSats.key(),
       &if self.index.index_sats {
@@ -736,22 +732,43 @@ impl<'index> Updater<'index> {
     txid: Txid,
     sat_to_satpoint: &mut Table<u64, &SatPointValue>,
     output_utxo_entries: &mut [UtxoEntryBuf],
-    input_sat_ranges: &mut VecDeque<(u64, u64)>,
+    input_sat_ranges: &[&[u8]],
+    leftover_sat_ranges: &mut Vec<u8>,
     sat_ranges_written: &mut u64,
     outputs_traversed: &mut u64,
   ) -> Result {
+    let mut pending_input_sat_range = None;
+    let mut input_sat_ranges_iter = input_sat_ranges
+      .iter()
+      .flat_map(|slice| slice.chunks_exact(11));
+
+    // Preallocate our temporary array, sized to hold the combined
+    // sat ranges from our inputs.  We'll never need more than that
+    // for a single output, even if we end up splitting some ranges.
+    let mut sats = Vec::with_capacity(
+      input_sat_ranges
+        .iter()
+        .map(|slice| slice.len())
+        .sum::<usize>(),
+    );
+
     for (vout, output) in tx.output.iter().enumerate() {
       let outpoint = OutPoint {
         vout: vout.try_into().unwrap(),
         txid,
       };
-      let mut sats = Vec::new();
 
       let mut remaining = output.value;
       while remaining > 0 {
-        let range = input_sat_ranges
-          .pop_front()
-          .ok_or_else(|| anyhow!("insufficient inputs for transaction outputs"))?;
+        let range = pending_input_sat_range.take().unwrap_or_else(|| {
+          SatRange::load(
+            input_sat_ranges_iter
+              .next()
+              .expect("insufficient inputs for transaction outputs")
+              .try_into()
+              .unwrap(),
+          )
+        });
 
         if !Sat(range.0).common() {
           sat_to_satpoint.insert(
@@ -769,7 +786,7 @@ impl<'index> Updater<'index> {
         let assigned = if count > remaining {
           self.sat_ranges_since_flush += 1;
           let middle = range.0 + remaining;
-          input_sat_ranges.push_front((middle, range.1));
+          pending_input_sat_range = Some((middle, range.1));
           (range.0, middle)
         } else {
           range
@@ -785,7 +802,13 @@ impl<'index> Updater<'index> {
       *outputs_traversed += 1;
 
       output_utxo_entries[vout].push_sat_ranges(&sats, self.index);
+      sats.clear();
+    }
+
+    if let Some(range) = pending_input_sat_range {
+      leftover_sat_ranges.extend(&range.store());
     }
+    leftover_sat_ranges.extend(input_sat_ranges_iter.flatten());
 
     Ok(())
   }

src/index/updater/inscription_updater.rs

@@ -67,7 +67,7 @@ impl<'a, 'tx> InscriptionUpdater<'a, 'tx> {
     output_utxo_entries: &mut [UtxoEntryBuf],
     utxo_cache: &mut HashMap<OutPoint, UtxoEntryBuf>,
     index: &Index,
-    input_sat_ranges: Option<&VecDeque<(u64, u64)>>,
+    input_sat_ranges: Option<&Vec<&[u8]>>,
   ) -> Result {
     let mut floating_inscriptions = Vec::new();
     let mut id_counter = 0;
@@ -340,14 +340,15 @@ impl<'a, 'tx> InscriptionUpdater<'a, 'tx> {
     }
   }
 
-  fn calculate_sat(
-    input_sat_ranges: Option<&VecDeque<(u64, u64)>>,
-    input_offset: u64,
-  ) -> Option<Sat> {
+  fn calculate_sat(input_sat_ranges: Option<&Vec<&[u8]>>, input_offset: u64) -> Option<Sat> {
     let input_sat_ranges = input_sat_ranges?;
 
     let mut offset = 0;
-    for (start, end) in input_sat_ranges {
+    for chunk in input_sat_ranges
+      .iter()
+      .flat_map(|slice| slice.chunks_exact(11))
+    {
+      let (start, end) = SatRange::load(chunk.try_into().unwrap());
       let size = end - start;
       if offset + size > input_offset {
         let n = start + input_offset - offset;
@@ -361,7 +362,7 @@ impl<'a, 'tx> InscriptionUpdater<'a, 'tx> {
 
   fn update_inscription_location(
     &mut self,
-    input_sat_ranges: Option<&VecDeque<(u64, u64)>>,
+    input_sat_ranges: Option<&Vec<&[u8]>>,
     flotsam: Flotsam,
     new_satpoint: SatPoint,
     op_return: bool,

src/lib.rs

@@ -63,7 +63,7 @@ use {
   std::{
     backtrace::BacktraceStatus,
     cmp,
-    collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque},
+    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
     env,
     ffi::OsString,
     fmt::{self, Display, Formatter},