Skip to content

Commit 5fba91c

Browse files
committed
refactor(napi/parser): raw transfer: introduce metadata struct (#12269)
Pure refactor. Introduce a `RawTransferMetadata` struct to hold metadata that gets written to end of buffer. Codegen use knowledge of memory layout of this struct to generate constants added in #12268. This is more robust than manually calculating offsets.
1 parent 39ef911 commit 5fba91c

File tree

8 files changed

+302
-271
lines changed

8 files changed

+302
-271
lines changed

crates/oxc_ast_macros/src/generated/structs.rs

Lines changed: 253 additions & 252 deletions
Large diffs are not rendered by default.

napi/parser/generated/constants.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33

44
const BUFFER_SIZE = 2147483648,
55
BUFFER_ALIGN = 4294967296,
6-
DATA_POINTER_POS_32 = 536870908,
7-
IS_TS_FLAG_POS = 2147483636,
6+
DATA_POINTER_POS_32 = 536870910,
7+
IS_TS_FLAG_POS = 2147483644,
88
PROGRAM_OFFSET = 0;
99

1010
module.exports = {

napi/parser/generated/deserialize/js.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ function deserialize(buffer, sourceTextInput, sourceLenInput) {
2020
sourceLen = sourceLenInput;
2121
sourceIsAscii = sourceText.length === sourceLen;
2222

23-
const data = deserializeRawTransferData(uint32[536870908]);
23+
const data = deserializeRawTransferData(uint32[536870910]);
2424

2525
uint8 =
2626
uint32 =

napi/parser/generated/deserialize/ts.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ function deserialize(buffer, sourceTextInput, sourceLenInput) {
2020
sourceLen = sourceLenInput;
2121
sourceIsAscii = sourceText.length === sourceLen;
2222

23-
const data = deserializeRawTransferData(uint32[536870908]);
23+
const data = deserializeRawTransferData(uint32[536870910]);
2424

2525
uint8 =
2626
uint32 =

napi/parser/src/generated/assert_layouts.rs

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,13 @@ const _: () = {
1717
assert!(offset_of!(RawTransferData, module) == 152);
1818
assert!(offset_of!(RawTransferData, errors) == 256);
1919

20+
// Padding: 3 bytes
21+
assert!(size_of::<RawTransferMetadata>() == 16);
22+
assert!(align_of::<RawTransferMetadata>() == 8);
23+
assert!(offset_of!(RawTransferMetadata, data_offset) == 8);
24+
assert!(offset_of!(RawTransferMetadata, is_ts) == 12);
25+
assert!(offset_of!(RawTransferMetadata, _padding) == 0);
26+
2027
// Padding: 7 bytes
2128
assert!(size_of::<Error>() == 80);
2229
assert!(align_of::<Error>() == 8);
@@ -68,6 +75,13 @@ const _: () = {
6875
assert!(offset_of!(RawTransferData, module) == 104);
6976
assert!(offset_of!(RawTransferData, errors) == 172);
7077

78+
// Padding: 3 bytes
79+
assert!(size_of::<RawTransferMetadata>() == 16);
80+
assert!(align_of::<RawTransferMetadata>() == 8);
81+
assert!(offset_of!(RawTransferMetadata, data_offset) == 8);
82+
assert!(offset_of!(RawTransferMetadata, is_ts) == 12);
83+
assert!(offset_of!(RawTransferMetadata, _padding) == 0);
84+
7185
// Padding: 3 bytes
7286
assert!(size_of::<Error>() == 44);
7387
assert!(align_of::<Error>() == 4);

napi/parser/src/raw_transfer.rs

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ use oxc_napi::get_source_type;
2020
use crate::{
2121
AstType, ParserOptions, get_ast_type, parse,
2222
raw_transfer_constants::{BUFFER_ALIGN, BUFFER_SIZE},
23-
raw_transfer_types::{EcmaScriptModule, Error, RawTransferData},
23+
raw_transfer_types::{EcmaScriptModule, Error, RawTransferData, RawTransferMetadata},
2424
};
2525

2626
// For raw transfer, use a buffer 2 GiB in size, with 4 GiB alignment.
@@ -179,10 +179,10 @@ unsafe fn parse_raw_impl(
179179
assert!(is_multiple_of(buffer_ptr as usize, BUFFER_ALIGN));
180180

181181
// Get offsets and size of data region to be managed by arena allocator.
182-
// Leave space for source before it, and 16 bytes for metadata after it.
182+
// Leave space for source before it, and space for metadata after it.
183183
// Metadata actually only takes 5 bytes, but round everything up to multiple of 16,
184184
// as `bumpalo` requires that alignment.
185-
const METADATA_SIZE: usize = 16;
185+
const METADATA_SIZE: usize = size_of::<RawTransferMetadata>();
186186
const {
187187
assert!(METADATA_SIZE >= BUMP_ALIGN);
188188
assert!(is_multiple_of(METADATA_SIZE, BUMP_ALIGN));
@@ -271,15 +271,16 @@ unsafe fn parse_raw_impl(
271271
ptr::from_ref(data).cast::<u8>()
272272
};
273273

274-
// Write offset of `RawTransferData` and `bool` representing AST type into end of buffer
274+
// Write metadata into end of buffer
275275
#[allow(clippy::cast_possible_truncation)]
276-
let data_offset = data_ptr as u32;
276+
let metadata = RawTransferMetadata::new(data_ptr as u32, ast_type == AstType::TypeScript);
277277
const METADATA_OFFSET: usize = BUFFER_SIZE - METADATA_SIZE;
278-
// SAFETY: `METADATA_OFFSET` is less than length of `buffer`
278+
const _: () = assert!(is_multiple_of(METADATA_OFFSET, BUMP_ALIGN));
279+
// SAFETY: `METADATA_OFFSET` is less than length of `buffer`.
280+
// `METADATA_OFFSET` is aligned on 16.
279281
#[expect(clippy::cast_ptr_alignment)]
280282
unsafe {
281-
buffer_ptr.add(METADATA_OFFSET).cast::<u32>().write(data_offset);
282-
buffer_ptr.add(METADATA_OFFSET + 4).cast::<bool>().write(ast_type == AstType::TypeScript);
283+
buffer_ptr.add(METADATA_OFFSET).cast::<RawTransferMetadata>().write(metadata);
283284
}
284285
}
285286

napi/parser/src/raw_transfer_types.rs

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,23 @@ pub struct RawTransferData<'a> {
2525
pub errors: Vec<'a, Error<'a>>,
2626
}
2727

28+
/// Metadata written to end of buffer.
29+
#[ast]
30+
pub struct RawTransferMetadata {
31+
/// Offset of `RawTransferData` within buffer.
32+
pub data_offset: u32,
33+
/// `true` if AST is TypeScript.
34+
pub is_ts: bool,
35+
/// Padding to pad struct to size 16.
36+
pub(crate) _padding: u64,
37+
}
38+
39+
impl RawTransferMetadata {
40+
pub fn new(data_offset: u32, is_ts: bool) -> Self {
41+
Self { data_offset, is_ts, _padding: 0 }
42+
}
43+
}
44+
2845
// Errors.
2946
//
3047
// These types and the `From` / `FromIn` impls mirror the implementation in `types.rs`

tasks/ast_tools/src/generators/raw_transfer.rs

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,6 @@ const BUFFER_SIZE: u32 = 1 << 31; // 2 GiB
3232
const _: () = assert!(BUFFER_SIZE % 16 == 0);
3333
/// Alignment of raw transfer buffer.
3434
const BUFFER_ALIGN: u64 = 1 << 32; // 4 GiB
35-
/// Size of metadata written to end of buffer.
36-
const METADATA_SIZE: u32 = 16;
37-
/// Offset of flag for whether AST is TypeScript or not, relative to start of metadata.
38-
const IS_TS_FLAG_OFFSET: u32 = 4;
3935

4036
// Offsets of `Vec`'s fields.
4137
// `Vec` is `#[repr(transparent)]` and `RawVec` is `#[repr(C)]`, so these offsets are fixed.
@@ -1020,9 +1016,11 @@ fn generate_constants(consts: Constants) -> (String, TokenStream) {
10201016

10211017
/// Calculate constants.
10221018
fn get_constants(schema: &Schema) -> Constants {
1023-
let metadata_pos = BUFFER_SIZE - METADATA_SIZE;
1024-
let data_pointer_pos_32 = metadata_pos / 4;
1025-
let is_ts_pos = metadata_pos + IS_TS_FLAG_OFFSET;
1019+
let metadata_struct = schema.type_by_name("RawTransferMetadata").as_struct().unwrap();
1020+
let metadata_pos = BUFFER_SIZE - metadata_struct.layout_64().size;
1021+
let data_pointer_pos = metadata_pos + metadata_struct.field_by_name("data_offset").offset_64();
1022+
let data_pointer_pos_32 = data_pointer_pos / 4;
1023+
let is_ts_pos = metadata_pos + metadata_struct.field_by_name("is_ts").offset_64();
10261024

10271025
let program_offset = schema
10281026
.type_by_name("RawTransferData")

0 commit comments

Comments
 (0)