Skip to content

Commit

Permalink
[vm] Extend AppJIT capability of gen_snapshot.
Browse files Browse the repository at this point in the history 
 - Add generation of AppJIT snapshots (from a trace rather than a training run).
 - Add generation of AppJIT snapshots under the constraint of not generating a new instructions segment.
 - Remove generation of script snapshots (no longer used by Flutter).

Change-Id: I2a7cb4b7cd681fae5d33bc896107b3ea903f35d1
Reviewed-on: https://dart-review.googlesource.com/75390
Commit-Queue: Ryan Macnak <rmacnak@google.com>
Reviewed-by: Stanislav Baranov <sbaranov@google.com>
  • Loading branch information
@rmacnak-google
rmacnak-google authored and commit-bot@chromium.org committed Sep 25, 2018
1 parent f81709c commit 86430cb
Show file tree
Hide file tree
Showing 8 changed files with 225 additions and 112 deletions.
213 changes: 114 additions & 99 deletions runtime/bin/gen_snapshot.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ static const int kErrorExitCode = 255;
}

// The core snapshot to use when creating isolates. Normally NULL, but loaded
// from a file when creating script snapshots.
// from a file when creating AppJIT snapshots.
const uint8_t* isolate_snapshot_data = NULL;
const uint8_t* isolate_snapshot_instructions = NULL;

Expand All @@ -69,7 +69,7 @@ const uint8_t* isolate_snapshot_instructions = NULL;
enum SnapshotKind {
kCore,
kCoreJIT,
kScript,
kAppJIT,
kAppAOTBlobs,
kAppAOTAssembly,
kVMAOTAssembly,
Expand All @@ -93,20 +93,26 @@ static bool ProcessEnvironmentOption(const char* arg,
static const char* kSnapshotKindNames[] = {
"core",
"core-jit",
"script",
"app-jit",
"app-aot-blobs",
"app-aot-assembly",
"vm-aot-assembly",
NULL,
};

#define STRING_OPTIONS_LIST(V) \
V(load_vm_snapshot_data, load_vm_snapshot_data_filename) \
V(load_vm_snapshot_instructions, load_vm_snapshot_instructions_filename) \
V(load_isolate_snapshot_data, load_isolate_snapshot_data_filename) \
V(load_isolate_snapshot_instructions, \
load_isolate_snapshot_instructions_filename) \
V(vm_snapshot_data, vm_snapshot_data_filename) \
V(vm_snapshot_instructions, vm_snapshot_instructions_filename) \
V(isolate_snapshot_data, isolate_snapshot_data_filename) \
V(isolate_snapshot_instructions, isolate_snapshot_instructions_filename) \
V(shared_data, shared_data_filename) \
V(shared_instructions, shared_instructions_filename) \
V(reused_instructions, reused_instructions_filename) \
V(assembly, assembly_filename) \
V(script_snapshot, script_snapshot_filename) \
V(dependencies, dependencies_filename) \
Expand Down Expand Up @@ -150,10 +156,6 @@ static bool IsSnapshottingForPrecompilation() {
(snapshot_kind == kVMAOTAssembly);
}

static bool SnapshotKindAllowedFromKernel() {
return snapshot_kind != kScript;
}

// clang-format off
static void PrintUsage() {
Log::PrintErr(
Expand Down Expand Up @@ -305,14 +307,17 @@ static int ParseArguments(int argc,
}
break;
}
case kScript: {
if ((vm_snapshot_data_filename == NULL) ||
case kAppJIT: {
if ((load_vm_snapshot_data_filename == NULL) ||
(isolate_snapshot_data_filename == NULL) ||
(script_snapshot_filename == NULL) || (*script_name == NULL)) {
((isolate_snapshot_instructions_filename == NULL) &&
(reused_instructions_filename == NULL))) {
Log::PrintErr(
"Building a script snapshot requires specifying input files for "
"--vm_snapshot_data and --isolate_snapshot_data, an output file "
"for --script_snapshot, and a Dart script.\n\n");
"Building an app JIT snapshot requires specifying input files for "
"--load_vm_snapshot_data and --load_vm_snapshot_instructions, an "
" output file for --isolate_snapshot_data, and either an output "
"file for --isolate_snapshot_instructions or an input file for "
"--reused_instructions.\n\n");
return -1;
}
break;
Expand Down Expand Up @@ -564,12 +569,13 @@ class DependenciesFileWriter : public ValueObject {
WriteDependenciesWithTarget(vm_snapshot_data_filename);
// WriteDependenciesWithTarget(isolate_snapshot_data_filename);
break;
case kScript:
WriteDependenciesWithTarget(script_snapshot_filename);
break;
case kAppAOTAssembly:
WriteDependenciesWithTarget(assembly_filename);
break;
case kAppJIT:
WriteDependenciesWithTarget(isolate_snapshot_data_filename);
// WriteDependenciesWithTarget(isolate_snapshot_instructions_filename);
break;
case kCoreJIT:
case kAppAOTBlobs:
WriteDependenciesWithTarget(vm_snapshot_data_filename);
Expand All @@ -593,21 +599,6 @@ class DependenciesFileWriter : public ValueObject {
WritePath(target);
Write(": ");

if (snapshot_kind == kScript) {
if (vm_snapshot_data_filename != NULL) {
WritePath(vm_snapshot_data_filename);
}
if (vm_snapshot_instructions_filename != NULL) {
WritePath(vm_snapshot_instructions_filename);
}
if (isolate_snapshot_data_filename != NULL) {
WritePath(isolate_snapshot_data_filename);
}
if (isolate_snapshot_instructions_filename != NULL) {
WritePath(isolate_snapshot_instructions_filename);
}
}

for (intptr_t i = 0; i < dependencies_->length(); i++) {
WritePath(dependencies_->At(i));
}
Expand Down Expand Up @@ -765,7 +756,7 @@ static Dart_Handle LoadGenericSnapshotCreationScript(

static void LoadCompilationTrace() {
if ((load_compilation_trace_filename != NULL) &&
(snapshot_kind == kCoreJIT)) {
((snapshot_kind == kCoreJIT) || (snapshot_kind == kAppJIT))) {
uint8_t* buffer = NULL;
intptr_t size = 0;
ReadFile(load_compilation_trace_filename, &buffer, &size);
Expand Down Expand Up @@ -805,6 +796,30 @@ static void CreateAndWriteCoreSnapshot() {
}
}

static std::unique_ptr<MappedMemory> MapFile(const char* filename,
File::MapType type,
const uint8_t** buffer) {
File* file = File::Open(NULL, filename, File::kRead);
if (file == NULL) {
Log::PrintErr("Failed to open: %s\n", filename);
exit(kErrorExitCode);
}
RefCntReleaseScope<File> rs(file);
intptr_t length = file->Length();
if (length == 0) {
// Can't map an empty file.
*buffer = NULL;
return NULL;
}
MappedMemory* mapping = file->Map(type, 0, length);
if (mapping == NULL) {
Log::PrintErr("Failed to read: %s\n", filename);
exit(kErrorExitCode);
}
*buffer = reinterpret_cast<const uint8_t*>(mapping->address());
return std::unique_ptr<MappedMemory>(mapping);
}

static void CreateAndWriteCoreJITSnapshot() {
ASSERT(snapshot_kind == kCoreJIT);
ASSERT(vm_snapshot_data_filename != NULL);
Expand Down Expand Up @@ -844,18 +859,38 @@ static void CreateAndWriteCoreJITSnapshot() {
isolate_snapshot_instructions_size);
}

static void CreateAndWriteScriptSnapshot() {
ASSERT(snapshot_kind == kScript);
ASSERT(script_snapshot_filename != NULL);
static void CreateAndWriteAppJITSnapshot() {
ASSERT(snapshot_kind == kAppJIT);
ASSERT(isolate_snapshot_data_filename != NULL);
ASSERT((isolate_snapshot_instructions_filename != NULL) ||
(reused_instructions_filename != NULL));

// First create a snapshot.
uint8_t* buffer = NULL;
intptr_t size = 0;
Dart_Handle result = Dart_CreateScriptSnapshot(&buffer, &size);
const uint8_t* reused_instructions = NULL;
std::unique_ptr<MappedMemory> mapped_reused_instructions;
if (reused_instructions_filename != NULL) {
mapped_reused_instructions = MapFile(reused_instructions_filename,
File::kReadOnly, &reused_instructions);
}

Dart_Handle result;
uint8_t* isolate_snapshot_data_buffer = NULL;
intptr_t isolate_snapshot_data_size = 0;
uint8_t* isolate_snapshot_instructions_buffer = NULL;
intptr_t isolate_snapshot_instructions_size = 0;

result = Dart_CreateAppJITSnapshotAsBlobs(
&isolate_snapshot_data_buffer, &isolate_snapshot_data_size,
&isolate_snapshot_instructions_buffer,
&isolate_snapshot_instructions_size, reused_instructions);
CHECK_RESULT(result);

// Now write it out to the specified file.
WriteFile(script_snapshot_filename, buffer, size);
WriteFile(isolate_snapshot_data_filename, isolate_snapshot_data_buffer,
isolate_snapshot_data_size);
if (reused_instructions_filename == NULL) {
WriteFile(isolate_snapshot_instructions_filename,
isolate_snapshot_instructions_buffer,
isolate_snapshot_instructions_size);
}
}

static void StreamingWriteCallback(void* callback_data,
Expand All @@ -870,30 +905,6 @@ static void StreamingWriteCallback(void* callback_data,
}
}

static std::unique_ptr<MappedMemory> MapFile(const char* filename,
File::MapType type,
const uint8_t** buffer) {
File* file = File::Open(NULL, filename, File::kRead);
if (file == NULL) {
Log::PrintErr("Failed to open: %s\n", filename);
exit(kErrorExitCode);
}
RefCntReleaseScope<File> rs(file);
intptr_t length = file->Length();
if (length == 0) {
// Can't map an empty file.
*buffer = NULL;
return NULL;
}
MappedMemory* mapping = file->Map(type, 0, length);
if (mapping == NULL) {
Log::PrintErr("Failed to read: %s\n", filename);
exit(kErrorExitCode);
}
*buffer = reinterpret_cast<const uint8_t*>(mapping->address());
return std::unique_ptr<MappedMemory>(mapping);
}

static void CreateAndWritePrecompiledSnapshot() {
ASSERT(IsSnapshottingForPrecompilation());
Dart_Handle result;
Expand Down Expand Up @@ -1050,8 +1061,6 @@ static Dart_QualifiedFunctionName no_entry_points[] = {

static int GenerateSnapshotFromKernel(const uint8_t* kernel_buffer,
intptr_t kernel_buffer_size) {
ASSERT(SnapshotKindAllowedFromKernel());

char* error = NULL;
IsolateData* isolate_data = new IsolateData(NULL, commandline_package_root,
commandline_packages_file, NULL);
Expand All @@ -1067,12 +1076,19 @@ static int GenerateSnapshotFromKernel(const uint8_t* kernel_buffer,
isolate_flags.entry_points = no_entry_points;
}

// We need to capture the vmservice library in the core snapshot, so load it
// in the main isolate as well.
isolate_flags.load_vmservice_library = true;
Dart_Isolate isolate = Dart_CreateIsolateFromKernel(
NULL, NULL, kernel_buffer, kernel_buffer_size, &isolate_flags,
isolate_data, &error);
Dart_Isolate isolate;
if (isolate_snapshot_data == NULL) {
// We need to capture the vmservice library in the core snapshot, so load it
// in the main isolate as well.
isolate_flags.load_vmservice_library = true;
isolate = Dart_CreateIsolateFromKernel(NULL, NULL, kernel_buffer,
kernel_buffer_size, &isolate_flags,
isolate_data, &error);
} else {
isolate = Dart_CreateIsolate(NULL, NULL, isolate_snapshot_data,
isolate_snapshot_instructions, NULL, NULL,
&isolate_flags, isolate_data, &error);
}
if (isolate == NULL) {
delete isolate_data;
Log::PrintErr("%s\n", error);
Expand Down Expand Up @@ -1125,6 +1141,10 @@ static int GenerateSnapshotFromKernel(const uint8_t* kernel_buffer,
LoadCompilationTrace();
CreateAndWriteCoreJITSnapshot();
break;
case kAppJIT:
LoadCompilationTrace();
CreateAndWriteAppJITSnapshot();
break;
default:
UNREACHABLE();
}
Expand Down Expand Up @@ -1172,13 +1192,6 @@ int main(int argc, char** argv) {
dfe.ReadScript(app_script_name, &kernel_buffer, &kernel_buffer_size);
}
if (kernel_buffer != NULL) {
if (!SnapshotKindAllowedFromKernel()) {
// TODO(sivachandra): Add check for the kernel program format (incremental
// vs batch).
Log::PrintErr(
"Can only generate core or aot snapshots from a kernel file.\n");
return kErrorExitCode;
}
if (dependencies_filename != NULL) {
Log::PrintErr("Depfiles are not supported in Dart 2.\n");
return kErrorExitCode;
Expand All @@ -1205,7 +1218,7 @@ int main(int argc, char** argv) {
if (IsSnapshottingForPrecompilation()) {
vm_options.AddArgument("--precompilation");
}
if (snapshot_kind == kCoreJIT) {
if (snapshot_kind == kCoreJIT || snapshot_kind == kAppJIT) {
vm_options.AddArgument("--fields_may_be_reset");
vm_options.AddArgument("--link_natives_lazily");
#if !defined(PRODUCT)
Expand Down Expand Up @@ -1246,26 +1259,25 @@ int main(int argc, char** argv) {
std::unique_ptr<MappedMemory> mapped_vm_snapshot_instructions;
std::unique_ptr<MappedMemory> mapped_isolate_snapshot_data;
std::unique_ptr<MappedMemory> mapped_isolate_snapshot_instructions;
if (snapshot_kind == kScript) {
if (load_vm_snapshot_data_filename != NULL) {
mapped_vm_snapshot_data =
MapFile(vm_snapshot_data_filename, File::kReadOnly,
MapFile(load_vm_snapshot_data_filename, File::kReadOnly,
&init_params.vm_snapshot_data);

if (vm_snapshot_instructions_filename != NULL) {
mapped_vm_snapshot_instructions =
MapFile(vm_snapshot_instructions_filename, File::kReadExecute,
&init_params.vm_snapshot_instructions);
}

}
if (load_vm_snapshot_instructions_filename != NULL) {
mapped_vm_snapshot_instructions =
MapFile(load_vm_snapshot_instructions_filename, File::kReadExecute,
&init_params.vm_snapshot_instructions);
}
if (load_isolate_snapshot_data_filename) {
mapped_isolate_snapshot_data =
MapFile(isolate_snapshot_data_filename, File::kReadOnly,
MapFile(load_isolate_snapshot_data_filename, File::kReadOnly,
&isolate_snapshot_data);

if (isolate_snapshot_instructions_filename != NULL) {
mapped_isolate_snapshot_instructions =
MapFile(isolate_snapshot_instructions_filename, File::kReadExecute,
&isolate_snapshot_instructions);
}
}
if (load_isolate_snapshot_instructions_filename != NULL) {
mapped_isolate_snapshot_instructions =
MapFile(load_isolate_snapshot_instructions_filename, File::kReadExecute,
&isolate_snapshot_instructions);
}

error = Dart_Initialize(&init_params);
Expand Down Expand Up @@ -1391,8 +1403,8 @@ int main(int argc, char** argv) {
case kCoreJIT:
CreateAndWriteCoreJITSnapshot();
break;
case kScript:
CreateAndWriteScriptSnapshot();
case kAppJIT:
CreateAndWriteAppJITSnapshot();
break;
case kAppAOTBlobs:
case kAppAOTAssembly:
Expand All @@ -1419,6 +1431,9 @@ int main(int argc, char** argv) {
case kCoreJIT:
CreateAndWriteCoreJITSnapshot();
break;
case kAppJIT:
CreateAndWriteAppJITSnapshot();
break;
default:
UNREACHABLE();
break;
Expand Down
4 changes: 2 additions & 2 deletions runtime/bin/snapshot_utils.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -368,7 +368,7 @@ void Snapshot::GenerateScript(const char* snapshot_filename) {

void Snapshot::GenerateAppJIT(const char* snapshot_filename) {
#if defined(TARGET_ARCH_IA32)
// Snapshots with code are not supported on IA32 or DBC.
// Snapshots with code are not supported on IA32.
uint8_t* isolate_buffer = NULL;
intptr_t isolate_size = 0;

Expand All @@ -387,7 +387,7 @@ void Snapshot::GenerateAppJIT(const char* snapshot_filename) {
intptr_t isolate_instructions_size = 0;
Dart_Handle result = Dart_CreateAppJITSnapshotAsBlobs(
&isolate_data_buffer, &isolate_data_size, &isolate_instructions_buffer,
&isolate_instructions_size);
&isolate_instructions_size, NULL);
if (Dart_IsError(result)) {
ErrorExit(kErrorExitCode, "%s\n", Dart_GetError(result));
}
Expand Down
3 changes: 2 additions & 1 deletion runtime/include/dart_api.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3470,7 +3470,8 @@ DART_EXPORT DART_WARN_UNUSED_RESULT Dart_Handle
Dart_CreateAppJITSnapshotAsBlobs(uint8_t** isolate_snapshot_data_buffer,
intptr_t* isolate_snapshot_data_size,
uint8_t** isolate_snapshot_instructions_buffer,
intptr_t* isolate_snapshot_instructions_size);
intptr_t* isolate_snapshot_instructions_size,
const uint8_t* reused_instructions);

/**
* Like Dart_CreateAppJITSnapshotAsBlobs, but also creates a new VM snapshot.
Expand Down
Loading

0 comments on commit 86430cb

Please sign in to comment.