Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add lazy_lsru_download flag to compute spec and automatically make decision about lazy SLRU download #11012

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
+108 −34
Open

Add lazy_lsru_download flag to compute spec and automatically make decision about lazy SLRU download #11012

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3709117
Add lazy_slru_download compute feature flag
Feb 26, 2025
7749af5
Add lazy_slru_download_threshold parameter to page server config
Feb 27, 2025
42fefef
Always update lazy_slru_download flag during basebackup
Feb 27, 2025
7c5ae84
Fix checking lazy SLRU download condition
Feb 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 22 additions & 23 deletions compute_tools/src/compute.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -386,30 +386,29 @@ impl ComputeNode {
let mut client = config.connect(NoTls)?;
let pageserver_connect_micros = start_time.elapsed().as_micros() as u64;

let replica = if spec.spec.mode != ComputeMode::Primary {
" --replica"
} else {
""
};
let lazy_slru_download = if spec
.spec
.features
.contains(&ComputeFeature::LazySlruDownload)
{
" --lazy-slru-download"
} else {
""
};
let basebackup_cmd = match lsn {
Lsn(0) => {
if spec.spec.mode != ComputeMode::Primary {
format!(
"basebackup {} {} --gzip --replica",
spec.tenant_id, spec.timeline_id
)
} else {
format!("basebackup {} {} --gzip", spec.tenant_id, spec.timeline_id)
}
}
_ => {
if spec.spec.mode != ComputeMode::Primary {
format!(
"basebackup {} {} {} --gzip --replica",
spec.tenant_id, spec.timeline_id, lsn
)
} else {
format!(
"basebackup {} {} {} --gzip",
spec.tenant_id, spec.timeline_id, lsn
)
}
}
Lsn(0) => format!(
"basebackup {} {} --gzip{}{}",
spec.tenant_id, spec.timeline_id, replica, lazy_slru_download
),
_ => format!(
"basebackup {} {} {} --gzip{}{}",
spec.tenant_id, spec.timeline_id, lsn, replica, lazy_slru_download
),
};

let copyreader = client.copy_out(basebackup_cmd.as_str())?;
Expand Down
3 changes: 3 additions & 0 deletions libs/compute_api/src/spec.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,9 @@ pub enum ComputeFeature {
/// Pre-install and initialize anon extension for every database in the cluster
AnonExtension,

/// Download larger SLU files on demand
LazySlruDownload,

/// This is a special feature flag that is used to represent unknown feature flags.
/// Basically all unknown to enum flags are represented as this one. See unit test
/// `parse_unknown_features()` for more details.
Expand Down
4 changes: 4 additions & 0 deletions libs/pageserver_api/src/config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,7 @@ pub struct ConfigToml {
pub enable_read_path_debugging: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub validate_wal_contiguity: Option<bool>,
pub lazy_slru_download_threshold: usize,
}

#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
Expand Down Expand Up @@ -418,6 +419,8 @@ pub mod defaults {

pub const DEFAULT_WAL_RECEIVER_PROTOCOL: utils::postgres_client::PostgresClientProtocol =
utils::postgres_client::PostgresClientProtocol::Vanilla;

pub const DEFAULT_LAZY_SLRU_DOWNLOAD_THRESHOLD: usize = 128;
}

impl Default for ConfigToml {
Expand Down Expand Up @@ -524,6 +527,7 @@ impl Default for ConfigToml {
None
},
validate_wal_contiguity: None,
lazy_slru_download_threshold: DEFAULT_LAZY_SLRU_DOWNLOAD_THRESHOLD,
}
}
}
Expand Down
18 changes: 14 additions & 4 deletions pageserver/src/basebackup.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ pub async fn send_basebackup_tarball<'a, W>(
prev_lsn: Option<Lsn>,
full_backup: bool,
replica: bool,
lazy_slru_download_enabled: bool,
ctx: &'a RequestContext,
) -> Result<(), BasebackupError>
where
Expand Down Expand Up @@ -112,8 +113,8 @@ where
};

info!(
"taking basebackup lsn={}, prev_lsn={} (full_backup={}, replica={})",
backup_lsn, prev_lsn, full_backup, replica
"taking basebackup lsn={}, prev_lsn={} (full_backup={}, replica={}, lazy_slru_download_enabled={})",
backup_lsn, prev_lsn, full_backup, replica, lazy_slru_download_enabled
);

let basebackup = Basebackup {
Expand All @@ -123,6 +124,7 @@ where
prev_record_lsn: prev_lsn,
full_backup,
replica,
lazy_slru_download_enabled,
ctx,
io_concurrency: IoConcurrency::spawn_from_conf(
timeline.conf,
Expand Down Expand Up @@ -151,6 +153,7 @@ where
prev_record_lsn: Lsn,
full_backup: bool,
replica: bool,
lazy_slru_download_enabled: bool,
ctx: &'a RequestContext,
io_concurrency: IoConcurrency,
}
Expand Down Expand Up @@ -289,7 +292,9 @@ where
self.timeline.pg_version,
)?;

let lazy_slru_download = self.timeline.get_lazy_slru_download() && !self.full_backup;
let lazy_slru_download = self.lazy_slru_download_enabled
&& self.timeline.get_lazy_slru_download()
&& !self.full_backup;

let pgversion = self.timeline.pg_version;
let subdirs = dispatch_pgversion!(pgversion, &pgv::bindings::PGDATA_SUBDIRS[..]);
Expand Down Expand Up @@ -333,20 +338,25 @@ where
);

let mut slru_builder = SlruSegmentsBuilder::new(&mut self.ar);

let mut total_slru_blocks = 0usize;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: The total size is already tracked in SlruSegmentBuilder. You could make SlruSegmentBuilder::finish return it instead of tracking it here again.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just think more about @hlinnaka second proposal - always include last segment of SLRU in basebackup.
My first though was that it is not so good idea, because in most cases Postgres doesn't need to access CLOG - it relies either on record hints, either on snapshot horizon.

But then I understand that to be able to update SRLU, we in any case need first to read updated segment (in most cases - last segment). So it may be really useful and reduce real startup time.

But then I recognised that there is not efficient way in current KV storage API to locate last segment (@VladLazar please correct me). So to locate last segment we in any case have to iterate through all segments. Which makes this optimization senseless.

My second thought was that there are many other pages which will be requested by backend in any case - some tables from system catalog. But we do not have such optimisation now and access this catalog pages in usual way.

Then I return back to first @hlinnaka proposal: always use on-demand download. If we are doing it for much more pages of system catalog, then doing it for few SLRU segments should not somehow significantly affect startup time.

Also, may be my first thought was also not so stupid: yes we need to download SLRU segment in order to update it. But we need to update SLRU if we perform some updates (write transaction). But most queries to the database are read-only, do not need XID and do not write something to SLRU. So using lazy download in this we can really reduce startup time even for small SLRUs (not sure how noticeable it will be).

So @hlinnaka , @VladLazar what do you think about removing all this logic with lazy SLRU download threshold and use it always (when it is enabled by control plane? Also may be remove it from tenant config?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But then I recognised that there is not efficient way in current KV storage API to locate last segment (@VladLazar please correct me). So to locate last segment we in any case have to iterate through all segments. Which makes this optimization senseless.

We don't have to iterate through all segments as far as I see. Let's consider a single slru kind for brevity:

  1. Read the slru segment directory key. This is an unordered set of segment indices.
  2. Pick the max out of the set from step (1)
  3. Read the segment size
  4. Vectored reads for the blocks in the segment. The max number of pages per segment is 32 (I think),
    so worst case we need two vectored reads (capped at 16 keys currently).

So overall, it adds up to 4 reads per slru kind.

what do you think about removing all this logic with lazy SLRU download threshold and use it always (when it is enabled by control plane? Also may be remove it from tenant config?

I don't have a lot of context as to when SLRU is needed inside the compute, but if total SLRU size is small why do extra round trips (PS <-> compute API only allows for one segment at a time)? Also, I think we should keep it as a tenant config to make it easier to test (both in regression tests and staging).

I just checked the on-demand slru segment download logic on the pageserver. It queries pages one by one, instead of using the vectorized API. We should update this regardless.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you!
I forget about SLRU directory.
Using this directory getting the last segment is really trivial operation.
But still not sure whether it make sense to include in basebackup just last segment or better to lazily download all segments.

for part in slru_partitions.parts {
let blocks = self
.timeline
.get_vectored(part, self.lsn, self.io_concurrency.clone(), self.ctx)
.await
.map_err(|e| BasebackupError::Server(e.into()))?;

total_slru_blocks += blocks.len();

for (key, block) in blocks {
let block = block.map_err(|e| BasebackupError::Server(e.into()))?;
slru_builder.add_block(&key, block).await?;
}
}
slru_builder.finish().await?;
self.timeline.set_lazy_slru_download(
total_slru_blocks > self.timeline.conf.lazy_slru_download_threshold,
);
}

let mut min_restart_lsn: Lsn = Lsn::MAX;
Expand Down
7 changes: 7 additions & 0 deletions pageserver/src/config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,11 @@ pub struct PageServerConf {
/// Interpreted protocol feature: if enabled, validate that the logical WAL received from
/// safekeepers does not have gaps.
pub validate_wal_contiguity: bool,

///
/// Size of SLRU object in blocks which triggers on-demand download rarther than including it in basebackup
///
pub lazy_slru_download_threshold: usize,
}

/// Token for authentication to safekeepers
Expand Down Expand Up @@ -365,6 +370,7 @@ impl PageServerConf {
get_vectored_concurrent_io,
enable_read_path_debugging,
validate_wal_contiguity,
lazy_slru_download_threshold,
} = config_toml;

let mut conf = PageServerConf {
Expand Down Expand Up @@ -452,6 +458,7 @@ impl PageServerConf {
no_sync: no_sync.unwrap_or(false),
enable_read_path_debugging: enable_read_path_debugging.unwrap_or(false),
validate_wal_contiguity: validate_wal_contiguity.unwrap_or(false),
lazy_slru_download_threshold,
};

// ------------------------------------------------------------
Expand Down
51 changes: 44 additions & 7 deletions pageserver/src/page_service.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2088,6 +2088,7 @@ impl PageServerHandler {
full_backup: bool,
gzip: bool,
replica: bool,
lazy_slru_download: bool,
ctx: &RequestContext,
) -> Result<(), QueryError>
where
Expand Down Expand Up @@ -2151,6 +2152,7 @@ impl PageServerHandler {
prev_lsn,
full_backup,
replica,
lazy_slru_download,
ctx,
)
.await
Expand All @@ -2174,6 +2176,7 @@ impl PageServerHandler {
prev_lsn,
full_backup,
replica,
lazy_slru_download,
ctx,
)
.await
Expand All @@ -2191,6 +2194,7 @@ impl PageServerHandler {
prev_lsn,
full_backup,
replica,
lazy_slru_download,
ctx,
)
.await
Expand Down Expand Up @@ -2240,14 +2244,15 @@ impl PageServerHandler {
}
}

/// `basebackup tenant timeline [lsn] [--gzip] [--replica]`
/// `basebackup tenant timeline [lsn] [--gzip] [--replica]`[--lazy-sru-download]
#[derive(Debug, Clone, Eq, PartialEq)]
struct BaseBackupCmd {
tenant_id: TenantId,
timeline_id: TimelineId,
lsn: Option<Lsn>,
gzip: bool,
replica: bool,
lazy_slru_download: bool,
}

/// `fullbackup tenant timeline [lsn] [prev_lsn]`
Expand Down Expand Up @@ -2380,6 +2385,7 @@ impl BaseBackupCmd {

let mut gzip = false;
let mut replica = false;
let mut lazy_slru_download = false;

for &param in &parameters[flags_parse_from..] {
match param {
Expand All @@ -2395,6 +2401,12 @@ impl BaseBackupCmd {
}
replica = true
}
"--lazy-slru-download" => {
if lazy_slru_download {
bail!("duplicate parameter for basebackup command: {param}")
}
lazy_slru_download = true
}
_ => bail!("invalid parameter for basebackup command: {param}"),
}
}
Expand All @@ -2404,6 +2416,7 @@ impl BaseBackupCmd {
lsn,
gzip,
replica,
lazy_slru_download,
})
}
}
Expand Down Expand Up @@ -2556,6 +2569,7 @@ where
lsn,
gzip,
replica,
lazy_slru_download,
}) => {
tracing::Span::current()
.record("tenant_id", field::display(tenant_id))
Expand All @@ -2577,6 +2591,7 @@ where
false,
gzip,
replica,
lazy_slru_download,
&ctx,
)
.await?;
Expand Down Expand Up @@ -2614,6 +2629,7 @@ where
true,
false,
false,
false,
&ctx,
)
.await?;
Expand Down Expand Up @@ -2747,7 +2763,8 @@ mod tests {
timeline_id,
lsn: None,
gzip: false,
replica: false
replica: false,
lazy_slru_download: false
})
);
let cmd =
Expand All @@ -2759,7 +2776,8 @@ mod tests {
timeline_id,
lsn: None,
gzip: true,
replica: false
replica: false,
lazy_slru_download: false
})
);
let cmd =
Expand All @@ -2771,7 +2789,8 @@ mod tests {
timeline_id,
lsn: None,
gzip: false,
replica: false
replica: false,
lazy_slru_download: false
})
);
let cmd = PageServiceCmd::parse(&format!("basebackup {tenant_id} {timeline_id} 0/16ABCDE"))
Expand All @@ -2783,7 +2802,8 @@ mod tests {
timeline_id,
lsn: Some(Lsn::from_str("0/16ABCDE").unwrap()),
gzip: false,
replica: false
replica: false,
lazy_slru_download: false
})
);
let cmd = PageServiceCmd::parse(&format!(
Expand All @@ -2797,7 +2817,23 @@ mod tests {
timeline_id,
lsn: None,
gzip: true,
replica: true
replica: true,
lazy_slru_download: false
})
);
let cmd = PageServiceCmd::parse(&format!(
"basebackup {tenant_id} {timeline_id} --replica --gzip --lazy-slru-download"
))
.unwrap();
assert_eq!(
cmd,
PageServiceCmd::BaseBackup(BaseBackupCmd {
tenant_id,
timeline_id,
lsn: None,
gzip: true,
replica: true,
lazy_slru_download: true
})
);
let cmd = PageServiceCmd::parse(&format!(
Expand All @@ -2811,7 +2847,8 @@ mod tests {
timeline_id,
lsn: Some(Lsn::from_str("0/16ABCDE").unwrap()),
gzip: true,
replica: true
replica: true,
lazy_slru_download: false
})
);
let cmd = PageServiceCmd::parse(&format!("fullbackup {tenant_id} {timeline_id}")).unwrap();
Expand Down
Loading
Loading