Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(static-files): break Arc reference cycle #6795

Merged
merged 26 commits into from
Feb 27, 2024
Merged
Show file tree
Hide file tree
Changes from 25 commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
3eae064
bump state tests timeout to 10m
shekhirin Feb 26, 2024
15b9b7d
revertme: display nextest config in ci
shekhirin Feb 26, 2024
4646da8
Revert "revertme: display nextest config in ci"
shekhirin Feb 26, 2024
8dd6b52
threads-required = 1
shekhirin Feb 26, 2024
5c1ec46
set NEXTEST_TEST_THREADS env var for state test
shekhirin Feb 26, 2024
1db9304
cleanup static files
shekhirin Feb 26, 2024
292e817
revertme: disable st_bad_opcode
shekhirin Feb 26, 2024
3065e53
revertme: print memory stats
shekhirin Feb 26, 2024
ec672cb
remove test-threads = 1
shekhirin Feb 26, 2024
5a564ba
ok time for swap
shekhirin Feb 26, 2024
3fc9b09
Revert "revertme: disable st_bad_opcode"
shekhirin Feb 26, 2024
c6fea2e
new swap space action
shekhirin Feb 26, 2024
737daa4
non-static writer
shekhirin Feb 26, 2024
2a895ef
add a dummy example
shekhirin Feb 26, 2024
b0acc7e
weak mf
shekhirin Feb 26, 2024
0154f30
improve minimal leak example, remove debug prints
shekhirin Feb 27, 2024
50e4e4f
cleanup debugging stuff, make weak beautiful
shekhirin Feb 27, 2024
08a7d44
restrict visibility of the inner static file provider field
shekhirin Feb 27, 2024
501a62e
remove swap
shekhirin Feb 27, 2024
b94d644
remove example demo
shekhirin Feb 27, 2024
8d5907a
document Weak
shekhirin Feb 27, 2024
0a5a9bd
fix test_snap test
shekhirin Feb 27, 2024
0b2978e
remove memory stats dep
shekhirin Feb 27, 2024
1d68d32
revert unit.yml
shekhirin Feb 27, 2024
03bfbde
expect parent static file provider
shekhirin Feb 27, 2024
181883c
one function to use everywhere
shekhirin Feb 27, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .config/nextest.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@ slow-timeout = { period = "30s", terminate-after = 4 }

[[profile.default.overrides]]
filter = "test(general_state_tests)"
slow-timeout = { period = "1m", terminate-after = 4 }
slow-timeout = { period = "1m", terminate-after = 10 }
2 changes: 2 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 5 additions & 3 deletions crates/stages/src/stages/execution.rs
Original file line number Diff line number Diff line change
Expand Up @@ -557,13 +557,14 @@ where

// Get next expected receipt number in static files
let static_file_provider = provider.static_file_provider();
let mut static_file_producer =
static_file_provider.get_writer(start_block, StaticFileSegment::Receipts)?;
let next_static_file_receipt_num = static_file_producer
let next_static_file_receipt_num = static_file_provider
.get_highest_static_file_tx(StaticFileSegment::Receipts)
.map(|num| num + 1)
.unwrap_or(0);

let mut static_file_producer =
static_file_provider.get_writer(start_block, StaticFileSegment::Receipts)?;

// Check if we had any unexpected shutdown after committing to static files, but
// NOT committing to database.
match next_static_file_receipt_num.cmp(&next_receipt_num) {
Expand All @@ -587,6 +588,7 @@ where
}
Ordering::Equal => {}
}

Ok(static_file_producer)
}

Expand Down
4 changes: 2 additions & 2 deletions crates/static-file/src/segments/headers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ impl<DB: Database> Segment<DB> for Headers {
block_range: RangeInclusive<BlockNumber>,
) -> ProviderResult<()> {
let range_len = block_range.clone().count();
let mut jar = prepare_jar::<DB, 3>(
let jar = prepare_jar::<DB, 3>(
provider,
directory,
StaticFileSegment::Headers,
Expand Down Expand Up @@ -119,7 +119,7 @@ impl<DB: Database> Segment<DB> for Headers {
None::<Vec<std::vec::IntoIter<Vec<u8>>>>,
hashes,
range_len,
&mut jar,
jar,
)?;

Ok(())
Expand Down
4 changes: 2 additions & 2 deletions crates/static-file/src/segments/receipts.rs
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ impl<DB: Database> Segment<DB> for Receipts {
let tx_range = provider.transaction_range_by_block_range(block_range.clone())?;
let tx_range_len = tx_range.clone().count();

let mut jar = prepare_jar::<DB, 1>(
let jar = prepare_jar::<DB, 1>(
provider,
directory,
StaticFileSegment::Receipts,
Expand Down Expand Up @@ -99,7 +99,7 @@ impl<DB: Database> Segment<DB> for Receipts {
None::<Vec<std::vec::IntoIter<Vec<u8>>>>,
hashes,
tx_range_len,
&mut jar,
jar,
)?;

Ok(())
Expand Down
4 changes: 2 additions & 2 deletions crates/static-file/src/segments/transactions.rs
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ impl<DB: Database> Segment<DB> for Transactions {
let tx_range = provider.transaction_range_by_block_range(block_range.clone())?;
let tx_range_len = tx_range.clone().count();

let mut jar = prepare_jar::<DB, 1>(
let jar = prepare_jar::<DB, 1>(
provider,
directory,
StaticFileSegment::Transactions,
Expand Down Expand Up @@ -103,7 +103,7 @@ impl<DB: Database> Segment<DB> for Transactions {
None::<Vec<std::vec::IntoIter<Vec<u8>>>>,
hashes,
tx_range_len,
&mut jar,
jar,
)?;

Ok(())
Expand Down
4 changes: 2 additions & 2 deletions crates/storage/db/src/static_file/generation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ macro_rules! generate_static_file_func {
dict_compression_set: Option<Vec<impl Iterator<Item = Vec<u8>>>>,
keys: Option<impl Iterator<Item = ColumnResult<impl PHFKey>>>,
row_count: usize,
nippy_jar: &mut NippyJar<H>
mut nippy_jar: NippyJar<H>
) -> ProviderResult<()>
where K: Key + Copy
{
Expand Down Expand Up @@ -88,7 +88,7 @@ macro_rules! generate_static_file_func {

debug!(target: "reth::static_file", jar=?nippy_jar, "Generating static file.");

nippy_jar.freeze(col_iterators.into_iter().chain(additional).collect(), row_count as u64)?;
let nippy_jar = nippy_jar.freeze(col_iterators.into_iter().chain(additional).collect(), row_count as u64)?;

debug!(target: "reth::static_file", jar=?nippy_jar, "Static file generated.");

Expand Down
Loading