Add Layer::overlaps method and use it in count_deltas to avoid unnecessary image layer generation

libs/pageserver_api/src/models.rs

@@ -241,6 +241,7 @@ pub struct LayerMapInfo {
 #[repr(usize)]
 pub enum LayerAccessKind {
  GetValueReconstructData,
+ ExtractHoles,
  Iter,
  KeyIter,
  Dump,

pageserver/benches/bench_layer_map.rs

@@ -1,5 +1,7 @@
+use pageserver::context::{DownloadBehavior, RequestContext};
 use pageserver::keyspace::{KeyPartitioning, KeySpace};
 use pageserver::repository::Key;
+use pageserver::task_mgr::TaskKind;
 use pageserver::tenant::layer_map::LayerMap;
 use pageserver::tenant::storage_layer::{Layer, LayerDescriptor, LayerFileName};
 use rand::prelude::{SeedableRng, SliceRandom, StdRng};
@@ -16,6 +18,7 @@ use utils::lsn::Lsn;
 use criterion::{criterion_group, criterion_main, Criterion};
 
 fn build_layer_map(filename_dump: PathBuf) -> LayerMap<LayerDescriptor> {
+ let ctx = RequestContext::new(TaskKind::Benchmark, DownloadBehavior::Error);
  let mut layer_map = LayerMap::<LayerDescriptor>::default();
 
  let mut min_lsn = Lsn(u64::MAX);
@@ -33,7 +36,7 @@ fn build_layer_map(filename_dump: PathBuf) -> LayerMap<LayerDescriptor> {
  min_lsn = min(min_lsn, lsn_range.start);
  max_lsn = max(max_lsn, Lsn(lsn_range.end.0 - 1));
 
- updates.insert_historic(Arc::new(layer));
+ updates.insert_historic(Arc::new(layer), &ctx).unwrap();
  }
 
  println!("min: {min_lsn}, max: {max_lsn}");
@@ -135,6 +138,7 @@ fn bench_from_captest_env(c: &mut Criterion) {
 // Benchmark using metadata extracted from a real project that was taknig
 // too long processing layer map queries.
 fn bench_from_real_project(c: &mut Criterion) {
+ let ctx = RequestContext::new(TaskKind::Benchmark, DownloadBehavior::Error);
  // Init layer map
  let now = Instant::now();
  let layer_map = build_layer_map(PathBuf::from("benches/odd-brook-layernames.txt"));
@@ -157,12 +161,13 @@ fn bench_from_real_project(c: &mut Criterion) {
  println!("running correctness check");
 
  let now = Instant::now();
- let result_bruteforce = layer_map.get_difficulty_map_bruteforce(latest_lsn, &partitioning);
+ let result_bruteforce =
+ layer_map.get_difficulty_map_bruteforce(latest_lsn, &partitioning, &ctx);
  assert!(result_bruteforce.len() == partitioning.parts.len());
  println!("Finished bruteforce in {:?}", now.elapsed());
 
  let now = Instant::now();
- let result_fast = layer_map.get_difficulty_map(latest_lsn, &partitioning, None);
+ let result_fast = layer_map.get_difficulty_map(latest_lsn, &partitioning, None, &ctx);
  assert!(result_fast.len() == partitioning.parts.len());
  println!("Finished fast in {:?}", now.elapsed());
 
@@ -189,14 +194,15 @@ fn bench_from_real_project(c: &mut Criterion) {
  });
  group.bench_function("get_difficulty_map", |b| {
  b.iter(|| {
- layer_map.get_difficulty_map(latest_lsn, &partitioning, Some(3));
+ layer_map.get_difficulty_map(latest_lsn, &partitioning, Some(3), &ctx);
  });
  });
  group.finish();
 }
 
 // Benchmark using synthetic data. Arrange image layers on stacked diagonal lines.
 fn bench_sequential(c: &mut Criterion) {
+ let ctx = RequestContext::new(TaskKind::Benchmark, DownloadBehavior::Error);
  // Init layer map. Create 100_000 layers arranged in 1000 diagonal lines.
  //
  // TODO This code is pretty slow and runs even if we're only running other
@@ -206,7 +212,7 @@ fn bench_sequential(c: &mut Criterion) {
  let now = Instant::now();
  let mut layer_map = LayerMap::default();
  let mut updates = layer_map.batch_update();
- for i in 0..100_000 {
+ for i in 1..100_000 {
  let i32 = (i as u32) % 100;
  let zero = Key::from_hex("000000000000000000000000000000000000").unwrap();
  let layer = LayerDescriptor {
@@ -215,7 +221,7 @@ fn bench_sequential(c: &mut Criterion) {
  is_incremental: false,
  short_id: format!("Layer {}", i),
  };
- updates.insert_historic(Arc::new(layer));
+ updates.insert_historic(Arc::new(layer), &ctx).unwrap();
  }
  updates.flush();
  println!("Finished layer map init in {:?}", now.elapsed());

pageserver/src/http/routes.rs

@@ -570,14 +570,15 @@ async fn layer_download_handler(request: Request<Body>) -> Result<Response<Body>
 }
 
 async fn evict_timeline_layer_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
+ let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Error);
  let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
  check_permission(&request, Some(tenant_id))?;
  let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
  let layer_file_name = get_request_param(&request, "layer_file_name")?;
 
  let timeline = active_timeline_of_active_tenant(tenant_id, timeline_id).await?;
  let evicted = timeline
- .evict_layer(layer_file_name)
+ .evict_layer(layer_file_name, &ctx)
  .await
  .map_err(ApiError::InternalServerError)?;
 

pageserver/src/task_mgr.rs

@@ -264,6 +264,8 @@ pub enum TaskKind {
 
  DebugTool,
 
+ Benchmark,
+
  #[cfg(test)]
  UnitTest,
 }

pageserver/src/tenant.rs

@@ -176,9 +176,9 @@ impl UninitializedTimeline<'_> {
  ///
  /// The new timeline is initialized in Active state, and its background jobs are
  /// started
- pub fn initialize(self, _ctx: &RequestContext) -> anyhow::Result<Arc<Timeline>> {
+ pub fn initialize(self, ctx: &RequestContext) -> anyhow::Result<Arc<Timeline>> {
  let mut timelines = self.owning_tenant.timelines.lock().unwrap();
- self.initialize_with_lock(&mut timelines, true, true)
+ self.initialize_with_lock(&mut timelines, true, true, ctx)
  }
 
  /// Like `initialize`, but the caller is already holding lock on Tenant::timelines.
@@ -191,6 +191,7 @@ impl UninitializedTimeline<'_> {
  timelines: &mut HashMap<TimelineId, Arc<Timeline>>,
  load_layer_map: bool,
  activate: bool,
+ ctx: &RequestContext,
  ) -> anyhow::Result<Arc<Timeline>> {
  let timeline_id = self.timeline_id;
  let tenant_id = self.owning_tenant.tenant_id;
@@ -211,7 +212,7 @@ impl UninitializedTimeline<'_> {
  Entry::Vacant(v) => {
  if load_layer_map {
  new_timeline
- .load_layer_map(new_disk_consistent_lsn)
+ .load_layer_map(new_disk_consistent_lsn, ctx)
  .with_context(|| {
  format!(
  "Failed to load layermap for timeline {tenant_id}/{timeline_id}"
@@ -459,7 +460,7 @@ impl Tenant {
  local_metadata: Option<TimelineMetadata>,
  ancestor: Option<Arc<Timeline>>,
  first_save: bool,
- _ctx: &RequestContext,
+ ctx: &RequestContext,
  ) -> anyhow::Result<()> {
  let tenant_id = self.tenant_id;
 
@@ -494,7 +495,7 @@ impl Tenant {
  // Do not start walreceiver here. We do need loaded layer map for reconcile_with_remote
  // But we shouldnt start walreceiver before we have all the data locally, because working walreceiver
  // will ingest data which may require looking at the layers which are not yet available locally
- match timeline.initialize_with_lock(&mut timelines_accessor, true, false) {
+ match timeline.initialize_with_lock(&mut timelines_accessor, true, false, ctx) {
  Ok(new_timeline) => new_timeline,
  Err(e) => {
  error!("Failed to initialize timeline {tenant_id}/{timeline_id}: {e:?}");
@@ -528,6 +529,7 @@ impl Tenant {
  .reconcile_with_remote(
  up_to_date_metadata,
  remote_startup_data.as_ref().map(|r| &r.index_part),
+ ctx,
  )
  .await
  .context("failed to reconcile with remote")?
@@ -1954,7 +1956,7 @@ impl Tenant {
  // made.
  break;
  }
- let result = timeline.gc().await?;
+ let result = timeline.gc(ctx).await?;
  totals += result;
  }
 
@@ -2078,7 +2080,7 @@ impl Tenant {
  src_timeline: &Arc<Timeline>,
  dst_id: TimelineId,
  start_lsn: Option<Lsn>,
- _ctx: &RequestContext,
+ ctx: &RequestContext,
  ) -> anyhow::Result<Arc<Timeline>> {
  let src_id = src_timeline.timeline_id;
 
@@ -2171,7 +2173,7 @@ impl Tenant {
  false,
  Some(Arc::clone(src_timeline)),
  )?
- .initialize_with_lock(&mut timelines, true, true)?;
+ .initialize_with_lock(&mut timelines, true, true, ctx)?;
  drop(timelines);
  info!("branched timeline {dst_id} from {src_id} at {start_lsn}");
 
@@ -2272,7 +2274,7 @@ impl Tenant {
 
  let timeline = {
  let mut timelines = self.timelines.lock().unwrap();
- raw_timeline.initialize_with_lock(&mut timelines, false, true)?
+ raw_timeline.initialize_with_lock(&mut timelines, false, true, ctx)?
  };
 
  info!(
@@ -3426,7 +3428,7 @@ mod tests {
  .await?;
  tline.freeze_and_flush().await?;
  tline.compact(&ctx).await?;
- tline.gc().await?;
+ tline.gc(&ctx).await?;
  }
 
  Ok(())
@@ -3498,7 +3500,7 @@ mod tests {
  .await?;
  tline.freeze_and_flush().await?;
  tline.compact(&ctx).await?;
- tline.gc().await?;
+ tline.gc(&ctx).await?;
  }
 
  Ok(())
@@ -3582,7 +3584,7 @@ mod tests {
  .await?;
  tline.freeze_and_flush().await?;
  tline.compact(&ctx).await?;
- tline.gc().await?;
+ tline.gc(&ctx).await?;
  }
 
  Ok(())