Skip to content

Commit

Permalink
bootstrap: make snapshot reproducible
Browse files Browse the repository at this point in the history
This patch uses the new V8 API to {de}serialize context slots for
snapshot in order to make the snapshot reproducible. Also
added a test for the reproducibility of snapshots.
  • Loading branch information
joyeecheung committed Mar 27, 2024
1 parent 759ee85 commit 9c8ab5e
Show file tree
Hide file tree
Showing 4 changed files with 121 additions and 3 deletions.
8 changes: 7 additions & 1 deletion src/api/environment.cc
Original file line number Diff line number Diff line change
Expand Up @@ -458,7 +458,13 @@ Environment* CreateEnvironment(
if (use_snapshot) {
context = Context::FromSnapshot(isolate,
SnapshotData::kNodeMainContextIndex,
{DeserializeNodeInternalFields, env})
v8::DeserializeInternalFieldsCallback(
DeserializeNodeInternalFields, env),
nullptr,
MaybeLocal<Value>(),
nullptr,
v8::DeserializeContextDataCallback(
DeserializeNodeContextData, env))
.ToLocalChecked();

CHECK(!context.IsEmpty());
Expand Down
56 changes: 54 additions & 2 deletions src/node_snapshotable.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1155,8 +1155,11 @@ ExitCode SnapshotBuilder::CreateSnapshot(SnapshotData* out,
CHECK_EQ(index, SnapshotData::kNodeVMContextIndex);
index = creator->AddContext(base_context);
CHECK_EQ(index, SnapshotData::kNodeBaseContextIndex);
index = creator->AddContext(main_context,
{SerializeNodeContextInternalFields, env});
index = creator->AddContext(
main_context,
v8::SerializeInternalFieldsCallback(SerializeNodeContextInternalFields,
env),
v8::SerializeContextDataCallback(SerializeNodeContextData, env));
CHECK_EQ(index, SnapshotData::kNodeMainContextIndex);
}

Expand Down Expand Up @@ -1255,6 +1258,17 @@ std::string SnapshotableObject::GetTypeName() const {
}
}

void DeserializeNodeContextData(Local<Context> holder,
int index,
StartupData payload,
void* callback_data) {
DCHECK(index == ContextEmbedderIndex::kEnvironment ||
index == ContextEmbedderIndex::kRealm ||
index == ContextEmbedderIndex::kContextTag);
// This is a no-op for now. We will reset all the pointers in
// Environment::AssignToContext() via the realm constructor.
}

void DeserializeNodeInternalFields(Local<Object> holder,
int index,
StartupData payload,
Expand Down Expand Up @@ -1320,6 +1334,44 @@ void DeserializeNodeInternalFields(Local<Object> holder,
}
}

StartupData SerializeNodeContextData(Local<Context> holder,
int index,
void* callback_data) {
DCHECK(index == ContextEmbedderIndex::kEnvironment ||
index == ContextEmbedderIndex::kContextifyContext ||
index == ContextEmbedderIndex::kRealm ||
index == ContextEmbedderIndex::kContextTag);
void* data = holder->GetAlignedPointerFromEmbedderData(index);
per_process::Debug(DebugCategory::MKSNAPSHOT,
"Serialize context data, index=%d, holder=%p, ptr=%p\n",
static_cast<int>(index),
*holder,
data);
// Serialization of contextify context is not yet supported.
if (index == ContextEmbedderIndex::kContextifyContext) {
DCHECK_NULL(data);
return {nullptr, 0};
}

// We need to use use new[] because V8 calls delete[] on the returned data.
int size = sizeof(ContextEmbedderIndex);
char* result = new char[size];
ContextEmbedderIndex* index_data =
reinterpret_cast<ContextEmbedderIndex*>(result);
*index_data = static_cast<ContextEmbedderIndex>(index);

// For now we just reset all of them in Environment::AssignToContext()
switch (index) {
case ContextEmbedderIndex::kEnvironment:
case ContextEmbedderIndex::kContextifyContext:
case ContextEmbedderIndex::kRealm:
case ContextEmbedderIndex::kContextTag:
return StartupData{result, size};
default:
UNREACHABLE();
}
}

StartupData SerializeNodeContextInternalFields(Local<Object> holder,
int index,
void* callback_data) {
Expand Down
7 changes: 7 additions & 0 deletions src/node_snapshotable.h
Original file line number Diff line number Diff line change
Expand Up @@ -126,10 +126,17 @@ class SnapshotableObject : public BaseObject {
v8::StartupData SerializeNodeContextInternalFields(v8::Local<v8::Object> holder,
int index,
void* env);
v8::StartupData SerializeNodeContextData(v8::Local<v8::Context> holder,
int index,
void* env);
void DeserializeNodeInternalFields(v8::Local<v8::Object> holder,
int index,
v8::StartupData payload,
void* env);
void DeserializeNodeContextData(v8::Local<v8::Context> holder,
int index,
v8::StartupData payload,
void* env);
void SerializeSnapshotableObjects(Realm* realm,
v8::SnapshotCreator* creator,
RealmSerializeInfo* info);
Expand Down
53 changes: 53 additions & 0 deletions test/parallel/test-snapshot-reproducible.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
'use strict';

require('../common');
const { spawnSyncAndExitWithoutError } = require('../common/child_process');
const tmpdir = require('../common/tmpdir');
const fs = require('fs');
const assert = require('assert');
const fixtures = require('../common/fixtures');

Check failure on line 8 in test/parallel/test-snapshot-reproducible.js

View workflow job for this annotation

GitHub Actions / lint-js-and-md

'fixtures' is assigned a value but never used

function generateSnapshot() {
tmpdir.refresh();

spawnSyncAndExitWithoutError(
process.execPath,
[
'--random_seed=42',
'--predictable',
'--build-snapshot',
'node:generate_default_snapshot',
],
{
cwd: tmpdir.path
}
);
const blobPath = tmpdir.resolve('snapshot.blob');
return fs.readFileSync(blobPath);
}

const buf1 = generateSnapshot();
const buf2 = generateSnapshot();
const diff = [];
let offset = 0;
const step = 16;
do {
const length = Math.min(buf1.length - offset, step);
const slice1 = buf1.slice(offset, offset + length).toString('hex');
const slice2 = buf2.slice(offset, offset + length).toString('hex');
if (slice1 != slice2) {

Check failure on line 38 in test/parallel/test-snapshot-reproducible.js

View workflow job for this annotation

GitHub Actions / lint-js-and-md

Expected '!==' and instead saw '!='
diff.push({offset, slice1, slice2});

Check failure on line 39 in test/parallel/test-snapshot-reproducible.js

View workflow job for this annotation

GitHub Actions / lint-js-and-md

A space is required after '{'

Check failure on line 39 in test/parallel/test-snapshot-reproducible.js

View workflow job for this annotation

GitHub Actions / lint-js-and-md

A space is required before '}'
}
offset += length;
} while (offset < buf1.length);

assert.strictEqual(offset, buf1.length);
if (offset < buf2.length) {
const length = Math.min(buf2.length - offset, step);
const slice2 = buf2.slice(offset, offset + length).toString('hex');
diff.push({offset, slice1: '', slice2});

Check failure on line 48 in test/parallel/test-snapshot-reproducible.js

View workflow job for this annotation

GitHub Actions / lint-js-and-md

A space is required after '{'

Check failure on line 48 in test/parallel/test-snapshot-reproducible.js

View workflow job for this annotation

GitHub Actions / lint-js-and-md

A space is required before '}'
offset += length;
} while (offset < buf2.length);

assert.deepStrictEqual(diff, [], 'Built-in snapshot should not change in different builds.');

Check failure on line 52 in test/parallel/test-snapshot-reproducible.js

View workflow job for this annotation

GitHub Actions / test-ubsan

--- stderr --- node:assert:126 throw new AssertionError(obj); ^ AssertionError [ERR_ASSERTION]: Built-in snapshot should not change in different builds. at Object.<anonymous> (/home/runner/work/node/node/test/parallel/test-snapshot-reproducible.js:52:8) at Module._compile (node:internal/modules/cjs/loader:1421:14) at Module._extensions..js (node:internal/modules/cjs/loader:1499:10) at Module.load (node:internal/modules/cjs/loader:1232:32) at Module._load (node:internal/modules/cjs/loader:1048:12) at Function.executeUserEntryPoint [as runMain] (node:internal/modules/run_main:187:14) at node:internal/main/run_main_module:28:49 { generatedMessage: false, code: 'ERR_ASSERTION', actual: [ { offset: 64, slice1: '000000a8b7df46e17a10b931312e392e', slice2: '000000aeb662c6e17a10b931312e392e' }, { offset: 1480464, slice1: '001101000805000011010460057bce02', slice2: '001101000805000011010460050b20cc' }, { offset: 1480480, slice1: '597f0000180000000000000032000000', slice2: '7f7f0000180000000000000032000000' }, { offset: 1480528, slice1: '00c9200008010000c92004c0017bce02', slice2: '00c9200008010000c92004c0010b20cc' }, { offset: 1480544, slice1: '597f000030000000000000002a000000', slice2: '7f7f000030000000000000002a000000' }, { offset: 1480656, slice1: '7cce02597f000018000000000000002f', slice2: '0c20cc7f7f000018000000000000002f' }, { offset: 1480752, slice1: '65044007ad08e6a05500001000000000', slice2: '650440073d6ffd215600001000000000' } ], expected: [], operator: 'deepStrictEqual' } Node.js v22.0.0-pre Command: out/Release/node --test-reporter=spec --test-reporter-destination=stdout --test-reporter=./tools/github_reporter/index.js --test-reporter-destination=stdout /home/runner/work/node/node/test/parallel/test-snapshot-reproducible.js

Check failure on line 52 in test/parallel/test-snapshot-reproducible.js

View workflow job for this annotation

GitHub Actions / test-linux

--- stderr --- node:assert:126 throw new AssertionError(obj); ^ AssertionError [ERR_ASSERTION]: Built-in snapshot should not change in different builds. at Object.<anonymous> (/home/runner/work/node/node/test/parallel/test-snapshot-reproducible.js:52:8) at Module._compile (node:internal/modules/cjs/loader:1421:14) at Module._extensions..js (node:internal/modules/cjs/loader:1499:10) at Module.load (node:internal/modules/cjs/loader:1232:32) at Module._load (node:internal/modules/cjs/loader:1048:12) at Function.executeUserEntryPoint [as runMain] (node:internal/modules/run_main:187:14) at node:internal/main/run_main_module:28:49 { generatedMessage: false, code: 'ERR_ASSERTION', actual: [ { offset: 64, slice1: '000000b4bedb82e17a10b931312e392e', slice2: '0000006cbee928e17a10b931312e392e' }, { offset: 1480464, slice1: '00110100080500001101046005acc153', slice2: '00110100080500001101046005ac6135' }, { offset: 1480480, slice1: '5c7f0000180000000000000032000000', slice2: 'd97f0000180000000000000032000000' }, { offset: 1480512, slice1: '066dad61050000001000000000000000', slice2: '06c7215e050000001000000000000000' }, { offset: 1480528, slice1: '00c9200008010000c92004c001acc153', slice2: '00c9200008010000c92004c001ac6135' }, { offset: 1480544, slice1: '5c7f000030000000000000002a000000', slice2: 'd97f000030000000000000002a000000' }, { offset: 1480592, slice1: '080800002d300440086ead6105000000', slice2: '080800002d30044008c7215e05000000' }, { offset: 1480656, slice1: 'acc1535c7f000018000000000000002f', slice2: 'ac6135d97f000018000000000000002f' }, { offset: 1480688, slice1: '650440043171b71f5600001000000000', slice2: '650440042e5142e75500001000000000' }, { offset: 1480720, slice1: '6ead6105000000180000000000000034', slice2: 'c7215e05000000180000000000000034' }, { offset: 1480752, slice1: '650440073371b71f5600001000000000', slice2: '650440072e5142e75500001000000000' } ], expected: [], operator: 'deepStrictEqual' } Node.js v22.0.0-pre Command: out/Release/node --test-reporter=spec --test-reporter-destination=stdout --test-reporter=./tools/github_reporter/index.js --test-reporter-destination=stdout /home/runner/work/node/node/test/parallel/test-snapshot-reproducible.js

Check failure on line 52 in test/parallel/test-snapshot-reproducible.js

View workflow job for this annotation

GitHub Actions / lint-js-and-md

Do not use a literal for the third argument of assert.deepStrictEqual()
assert.strictEqual(buf1.length, buf2.length);

0 comments on commit 9c8ab5e

Please sign in to comment.