Skip to content

Commit a15225f

Browse files
sokrasokra
committed
Turbopack: remove value compression dictionary
It doesn't have benefit as each block is already large as we merge small values together and medium size values are already large. This also gives the benefit that we do not need to recompress medium value blocks when doing compaction
1 parent a2cb74e commit a15225f

File tree

10 files changed

+85
-162
lines changed

10 files changed

+85
-162
lines changed

‎turbopack/crates/turbo-persistence-tools/src/main.rs‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@ fn main() -> Result<()> {
3535
amqf_entries,
3636
sst_size,
3737
key_compression_dictionary_size,
38-
value_compression_dictionary_size,
3938
block_count,
4039
} in meta_file.entries
4140
{
@@ -45,15 +44,11 @@ fn main() -> Result<()> {
4544
);
4645
println!(" AMQF {amqf_entries} entries = {} KiB", amqf_size / 1024);
4746
println!(
48-
" {} KiB = {} kiB key compression dict + {} KiB value compression dict + \
49-
{block_count} blocks (avg {} bytes/block)",
47+
" {} KiB = {} kiB key compression dict + {block_count} blocks (avg {} \
48+
bytes/block)",
5049
sst_size / 1024,
5150
key_compression_dictionary_size / 1024,
52-
value_compression_dictionary_size / 1024,
53-
(sst_size
54-
- key_compression_dictionary_size as u64
55-
- value_compression_dictionary_size as u64)
56-
/ block_count as u64
51+
(sst_size - key_compression_dictionary_size as u64) / block_count as u64
5752
);
5853
}
5954
if !meta_file.obsolete_sst_files.is_empty() {

‎turbopack/crates/turbo-persistence/README.md‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,6 @@ A meta file can contain metadata about multiple SST files. The metadata is store
4545
- foreach described SST file
4646
- 4 bytes sequence number of the SST file
4747
- 2 bytes key Compression Dictionary length
48-
- 2 bytes value Compression Dictionary length
4948
- 2 bytes block count
5049
- 8 bytes min hash
5150
- 8 bytes max hash
@@ -59,7 +58,6 @@ A meta file can contain metadata about multiple SST files. The metadata is store
5958
The SST file contains only data without any header.
6059

6160
- serialized key Compression Dictionary
62-
- serialized value Compression Dictionary
6361
- foreach block
6462
- 4 bytes uncompressed block length
6563
- compressed data

‎turbopack/crates/turbo-persistence/src/collector.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -92,11 +92,11 @@ impl<K: StoreKey, const SIZE_SHIFT: usize> Collector<K, SIZE_SHIFT> {
9292
self.entries.push(entry);
9393
}
9494

95-
/// Sorts the entries and returns them along with the total key and value sizes. This doesn't
95+
/// Sorts the entries and returns them along with the total key size. This doesn't
9696
/// clear the entries.
97-
pub fn sorted(&mut self) -> (&[CollectorEntry<K>], usize, usize) {
97+
pub fn sorted(&mut self) -> (&[CollectorEntry<K>], usize) {
9898
self.entries.sort_unstable_by(|a, b| a.key.cmp(&b.key));
99-
(&self.entries, self.total_key_size, self.total_value_size)
99+
(&self.entries, self.total_key_size)
100100
}
101101

102102
/// Clears the collector.

‎turbopack/crates/turbo-persistence/src/db.rs‎

Lines changed: 10 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -904,8 +904,6 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
904904
amqf,
905905
key_compression_dictionary_length: entry
906906
.key_compression_dictionary_length(),
907-
value_compression_dictionary_length: entry
908-
.value_compression_dictionary_length(),
909907
block_count: entry.block_count(),
910908
size: entry.size(),
911909
entries: 0,
@@ -919,7 +917,6 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
919917
fn create_sst_file<'l>(
920918
entries: &[LookupEntry<'l>],
921919
total_key_size: usize,
922-
total_value_size: usize,
923920
path: &Path,
924921
seq: u32,
925922
) -> Result<(u32, File, StaticSortedFileBuilderMeta<'static>)>
@@ -929,7 +926,6 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
929926
let (meta, file) = write_static_stored_file(
930927
entries,
931928
total_key_size,
932-
total_value_size,
933929
&path.join(format!("{seq:08}.sst")),
934930
)?;
935931
Ok((seq, file, meta))
@@ -963,7 +959,7 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
963959
let mut current: Option<LookupEntry<'_>> = None;
964960
let mut entries = Vec::new();
965961
let mut last_entries = Vec::new();
966-
let mut last_entries_total_sizes = (0, 0);
962+
let mut last_entries_total_key_size = 0;
967963
for entry in iter {
968964
let entry = entry?;
969965

@@ -980,15 +976,11 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
980976
> DATA_THRESHOLD_PER_COMPACTED_FILE
981977
|| entries.len() >= MAX_ENTRIES_PER_COMPACTED_FILE
982978
{
983-
let (
984-
selected_total_key_size,
985-
selected_total_value_size,
986-
) = last_entries_total_sizes;
979+
let selected_total_key_size =
980+
last_entries_total_key_size;
987981
swap(&mut entries, &mut last_entries);
988-
last_entries_total_sizes = (
989-
total_key_size - key_size,
990-
total_value_size - value_size,
991-
);
982+
last_entries_total_key_size =
983+
total_key_size - key_size;
992984
total_key_size = key_size;
993985
total_value_size = value_size;
994986

@@ -1001,7 +993,6 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
1001993
new_sst_files.push(create_sst_file(
1002994
&entries,
1003995
selected_total_key_size,
1004-
selected_total_value_size,
1005996
path,
1006997
seq,
1007998
)?);
@@ -1019,7 +1010,8 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
10191010
}
10201011
if let Some(entry) = current {
10211012
total_key_size += entry.key.len();
1022-
total_value_size += entry.value.uncompressed_size_in_sst();
1013+
// Obsolete as we no longer need total_value_size
1014+
// total_value_size += entry.value.uncompressed_size_in_sst();
10231015
entries.push(entry);
10241016
}
10251017

@@ -1031,7 +1023,6 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
10311023
new_sst_files.push(create_sst_file(
10321024
&entries,
10331025
total_key_size,
1034-
total_value_size,
10351026
path,
10361027
seq,
10371028
)?);
@@ -1042,8 +1033,7 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
10421033
if !last_entries.is_empty() {
10431034
last_entries.append(&mut entries);
10441035

1045-
last_entries_total_sizes.0 += total_key_size;
1046-
last_entries_total_sizes.1 += total_value_size;
1036+
last_entries_total_key_size += total_key_size;
10471037

10481038
let (part1, part2) =
10491039
last_entries.split_at(last_entries.len() / 2);
@@ -1055,17 +1045,15 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
10551045
new_sst_files.push(create_sst_file(
10561046
part1,
10571047
// We don't know the exact sizes so we estimate them
1058-
last_entries_total_sizes.0 / 2,
1059-
last_entries_total_sizes.1 / 2,
1048+
last_entries_total_key_size / 2,
10601049
path,
10611050
seq1,
10621051
)?);
10631052

10641053
keys_written += part2.len() as u64;
10651054
new_sst_files.push(create_sst_file(
10661055
part2,
1067-
last_entries_total_sizes.0 / 2,
1068-
last_entries_total_sizes.1 / 2,
1056+
last_entries_total_key_size / 2,
10691057
path,
10701058
seq2,
10711059
)?);
@@ -1265,8 +1253,6 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
12651253
amqf_entries: amqf.len(),
12661254
key_compression_dictionary_size: entry
12671255
.key_compression_dictionary_length(),
1268-
value_compression_dictionary_size: entry
1269-
.value_compression_dictionary_length(),
12701256
block_count: entry.block_count(),
12711257
}
12721258
})
@@ -1304,6 +1290,5 @@ pub struct MetaFileEntryInfo {
13041290
pub amqf_entries: usize,
13051291
pub sst_size: u64,
13061292
pub key_compression_dictionary_size: u16,
1307-
pub value_compression_dictionary_size: u16,
13081293
pub block_count: u16,
13091294
}

‎turbopack/crates/turbo-persistence/src/lookup_entry.rs‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ pub enum LazyLookupValue<'l> {
2222
Medium {
2323
uncompressed_size: u32,
2424
block: &'l [u8],
25-
dictionary: &'l [u8],
2625
},
2726
}
2827

@@ -79,11 +78,9 @@ impl Entry for LookupEntry<'_> {
7978
LazyLookupValue::Medium {
8079
uncompressed_size,
8180
block,
82-
dictionary,
8381
} => EntryValue::MediumCompressed {
8482
uncompressed_size: *uncompressed_size,
8583
block,
86-
dictionary,
8784
},
8885
}
8986
}

‎turbopack/crates/turbo-persistence/src/meta_file.rs‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -144,10 +144,6 @@ impl MetaEntry {
144144
self.sst_data.key_compression_dictionary_length
145145
}
146146

147-
pub fn value_compression_dictionary_length(&self) -> u16 {
148-
self.sst_data.value_compression_dictionary_length
149-
}
150-
151147
pub fn block_count(&self) -> u16 {
152148
self.sst_data.block_count
153149
}
@@ -222,7 +218,6 @@ impl MetaFile {
222218
sst_data: StaticSortedFileMetaData {
223219
sequence_number: file.read_u32::<BE>()?,
224220
key_compression_dictionary_length: file.read_u16::<BE>()?,
225-
value_compression_dictionary_length: file.read_u16::<BE>()?,
226221
block_count: file.read_u16::<BE>()?,
227222
},
228223
family,

‎turbopack/crates/turbo-persistence/src/meta_file_builder.rs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,6 @@ impl<'a> MetaFileBuilder<'a> {
5858
for (sequence_number, sst) in &self.entries {
5959
file.write_u32::<BE>(*sequence_number)?;
6060
file.write_u16::<BE>(sst.key_compression_dictionary_length)?;
61-
file.write_u16::<BE>(sst.value_compression_dictionary_length)?;
6261
file.write_u16::<BE>(sst.block_count)?;
6362
file.write_u64::<BE>(sst.min_hash)?;
6463
file.write_u64::<BE>(sst.max_hash)?;

‎turbopack/crates/turbo-persistence/src/static_sorted_file.rs‎

Lines changed: 25 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ use std::{
1010

1111
use anyhow::{Context, Result, bail};
1212
use byteorder::{BE, ReadBytesExt};
13-
use lzzzz::lz4::decompress_with_dict;
13+
use lzzzz::lz4::{decompress, decompress_with_dict};
1414
use memmap2::Mmap;
1515
use quick_cache::sync::GuardResult;
1616
use rustc_hash::FxHasher;
@@ -67,8 +67,6 @@ pub struct StaticSortedFileMetaData {
6767
pub sequence_number: u32,
6868
/// The length of the key compression dictionary.
6969
pub key_compression_dictionary_length: u16,
70-
/// The length of the value compression dictionary.
71-
pub value_compression_dictionary_length: u16,
7270
/// The number of blocks in the SST file.
7371
pub block_count: u16,
7472
}
@@ -81,21 +79,14 @@ impl StaticSortedFileMetaData {
8179

8280
pub fn blocks_start(&self) -> usize {
8381
let k: usize = self.key_compression_dictionary_length.into();
84-
let v: usize = self.value_compression_dictionary_length.into();
85-
k + v
82+
k
8683
}
8784

8885
pub fn key_compression_dictionary_range(&self) -> Range<usize> {
8986
let start = 0;
9087
let end: usize = self.key_compression_dictionary_length.into();
9188
start..end
9289
}
93-
94-
pub fn value_compression_dictionary_range(&self) -> Range<usize> {
95-
let start = self.key_compression_dictionary_length as usize;
96-
let end = start + self.value_compression_dictionary_length as usize;
97-
start..end
98-
}
9990
}
10091

10192
/// A memory mapped SST file.
@@ -323,25 +314,34 @@ impl StaticSortedFile {
323314

324315
/// Reads a key block from the file.
325316
fn read_key_block(&self, block_index: u16) -> Result<ArcSlice<u8>> {
326-
self.read_block(
317+
self.read_block_with_dict(
327318
block_index,
328319
&self.mmap[self.meta.key_compression_dictionary_range()],
329320
)
330321
}
331322

332323
/// Reads a value block from the file.
333324
fn read_value_block(&self, block_index: u16) -> Result<ArcSlice<u8>> {
334-
self.read_block(
335-
block_index,
336-
&self.mmap[self.meta.value_compression_dictionary_range()],
337-
)
325+
self.read_block(block_index)
326+
}
327+
328+
/// Reads a block from the file.
329+
fn read_block_with_dict(
330+
&self,
331+
block_index: u16,
332+
compression_dictionary: &[u8],
333+
) -> Result<ArcSlice<u8>> {
334+
let (uncompressed_length, block) = self.get_compressed_block(block_index)?;
335+
336+
let buffer = decompress_into_arc(uncompressed_length, block, Some(compression_dictionary))?;
337+
Ok(ArcSlice::from(buffer))
338338
}
339339

340340
/// Reads a block from the file.
341-
fn read_block(&self, block_index: u16, compression_dictionary: &[u8]) -> Result<ArcSlice<u8>> {
341+
fn read_block(&self, block_index: u16) -> Result<ArcSlice<u8>> {
342342
let (uncompressed_length, block) = self.get_compressed_block(block_index)?;
343343

344-
let buffer = decompress_into_arc(uncompressed_length, block, compression_dictionary)?;
344+
let buffer = decompress_into_arc(uncompressed_length, block, None)?;
345345
Ok(ArcSlice::from(buffer))
346346
}
347347

@@ -486,8 +486,6 @@ impl<'l> StaticSortedFileIter<'l> {
486486
LazyLookupValue::Medium {
487487
uncompressed_size,
488488
block,
489-
dictionary: &self.this.mmap
490-
[self.this.meta.value_compression_dictionary_range()],
491489
}
492490
} else {
493491
let value = self
@@ -587,10 +585,10 @@ fn get_key_entry<'l>(
587585
})
588586
}
589587

590-
pub fn decompress_into_arc(
588+
fn decompress_into_arc(
591589
uncompressed_length: u32,
592590
block: &[u8],
593-
compression_dictionary: &[u8],
591+
compression_dictionary: Option<&[u8]>,
594592
) -> Result<Arc<[u8]>> {
595593
// We directly allocate the buffer in an Arc to avoid copying it into an Arc and avoiding
596594
// double indirection. This is a dynamically sized arc.
@@ -602,7 +600,11 @@ pub fn decompress_into_arc(
602600
// Safety: We know that the buffer is not shared yet.
603601
let decompressed = unsafe { Arc::get_mut_unchecked(&mut buffer) };
604602
// Safety: decompress_with_dict will only write to `decompressed` and not read from it.
605-
let bytes_writes = decompress_with_dict(block, decompressed, compression_dictionary)?;
603+
let bytes_writes = if let Some(dict) = compression_dictionary {
604+
decompress_with_dict(&block, decompressed, dict)?
605+
} else {
606+
decompress(&block, decompressed)?
607+
};
606608
assert_eq!(
607609
bytes_writes, uncompressed_length as usize,
608610
"Decompressed length does not match expected length"

0 commit comments

Comments
 (0)