Skip to content

Commit

Permalink
src: further simplify large pages code
Browse files Browse the repository at this point in the history 
* Clean up and co-locate explanation for what the code does to the top
  of the file.
* Remove extraneous headers.
* Remove the need for `OnScopeLeave` replacing it with a pair of labels
  `fail:` and `done:`.

Re: nodejs#32570
Signed-off-by: Gabriel Schulhof <gabriel.schulhof@intel.com>
  • Loading branch information
Gabriel Schulhof committed Mar 31, 2020
1 parent 67d5c90 commit c9b1139
Showing 1 changed file with 48 additions and 47 deletions.
95 changes: 48 additions & 47 deletions src/large_pages/node_large_page.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,22 +20,56 @@
//
// SPDX-License-Identifier: MIT

// The functions in this file map the .text section of Node.js into 2MB pages.
// They perform the following steps:
//
// 1: Find the Node.js binary's `.text` section in memory. This is done below in
// `FindNodeTextRegion`. It is accomplished in a platform-specific way. On
// Linux and FreeBSD, `dl_iterate_phdr(3)` is used. When the region is found,
// it is "trimmed" as follows:
// * Modify the start to point to the very beginning of node `.text` section
// (from symbol `__node_text_start` declared in node_text_start.S).
// * Possibly modify the end to account for the `lpstub` section which
// contains `MoveTextRegionToLargePages`, the function we do not wish to
// move (see below).
// * Align the address of the start to its nearest higher large page
// boundary.
// * Align the address of the end to its nearest lower large page boundaries.
//
// 2: Move the text region to large pages. This is done below in
// `MoveTextRegionToLargePages`. We need to be very careful:
// a) `MoveTextRegionToLargePages` itself should not be moved.
// We use gcc attributes
// (__section__) to put it outside the `.text` section,
// (__aligned__) to align it at the 2M boundary, and
// (__noline__) to not inline this function.
// b) `MoveTextRegionToLargePages` should not call any function(s) that might
// be moved.
// To move the .text section, perform the following steps:
// * Map a new, temporary area and copy the original code there.
// * Use mmap using the start address with MAP_FIXED so we get exactly the
// same virtual address (except on OSX). On platforms other than Linux,
// use mmap flags to request hugepages.
// * On Linux use madvise with MADV_HUGEPAGE to use anonymous 2MB pages.
// * If successful copy the code to the newly mapped area and protect it to
// be readable and executable.
// * Unmap the temporary area.

#include "node_large_page.h"

#include <cerrno> // NOLINT(build/include)

// Besides returning ENOTSUP at runtime we do nothing if this define is missing.
#if defined(NODE_ENABLE_LARGE_CODE_PAGES) && NODE_ENABLE_LARGE_CODE_PAGES
#include "debug_utils-inl.h"
#include "util.h"
#include "uv.h"

#if defined(__linux__) || defined(__FreeBSD__)
#include <string.h>
#if defined(__linux__)
#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif // ifndef _GNU_SOURCE
#elif defined(__FreeBSD__)
#include "uv.h" // uv_exepath
#endif // defined(__linux__)
#include <link.h>
#endif // defined(__linux__) || defined(__FreeBSD__)
Expand All @@ -44,38 +78,16 @@
#include <sys/mman.h>
#if defined(__FreeBSD__)
#include <sys/sysctl.h>
#include <sys/user.h>
#elif defined(__APPLE__)
#include <mach/vm_map.h>
#endif
#include <unistd.h> // getpid

#include <climits> // PATH_MAX
#include <clocale>
#include <csignal>
#include <cstdlib>
#include <cstdint>
#include <cstring>
#include <string>
#include <fstream>
#include <iostream>
#include <vector>

// The functions in this file map the text segment of node into 2M pages.
// The algorithm is simple
// Find the text region of node binary in memory
// 1: Examine the /proc/self/maps to determine the currently mapped text
// region and obtain the start and end
// Modify the start to point to the very beginning of node text segment
// (from variable nodetext setup in ld.script)
// Align the address of start and end to Large Page Boundaries
//
// 2: Move the text region to large pages
// Map a new area and copy the original code there
// Use mmap using the start address with MAP_FIXED so we get exactly the
// same virtual address
// Use madvise with MADV_HUGEPAGE to use Anonymous 2M Pages
// If successful copy the code there and unmap the original region.

#if defined(__linux__) || defined(__FreeBSD__)
extern "C" {
Expand Down Expand Up @@ -284,18 +296,6 @@ bool IsSuperPagesEnabled() {

} // End of anonymous namespace

// Moving the text region to large pages. We need to be very careful.
// 1: This function itself should not be moved.
// We use a gcc attributes
// (__section__) to put it outside the ".text" section
// (__aligned__) to align it at 2M boundary
// (__noline__) to not inline this function
// 2: This function should not call any function(s) that might be moved.
// a. map a new area and copy the original code there
// b. mmap using the start address with MAP_FIXED so we get exactly
// the same virtual address (except on macOS).
// c. madvise with MADV_HUGEPAGE
// d. If successful copy the code there and unmap the original region
int
#if !defined(__APPLE__)
__attribute__((__section__("lpstub")))
Expand All @@ -305,18 +305,12 @@ __attribute__((__section__("__TEXT,__lpstub")))
__attribute__((__aligned__(hps)))
__attribute__((__noinline__))
MoveTextRegionToLargePages(const text_region& r) {
int status = 0;
void* nmem = nullptr;
void* tmem = nullptr;
void* start = r.from;
size_t size = r.to - r.from;

auto free_mems = OnScopeLeave([&nmem, &tmem, size]() {
if (nmem != nullptr && nmem != MAP_FAILED && munmap(nmem, size) == -1)
PrintSystemError(errno);
if (tmem != nullptr && tmem != MAP_FAILED && munmap(tmem, size) == -1)
PrintSystemError(errno);
});

// Allocate temporary region and back up the code we will re-map.
nmem = mmap(nullptr, size,
PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
Expand Down Expand Up @@ -359,12 +353,19 @@ MoveTextRegionToLargePages(const text_region& r) {
#endif

if (mprotect(start, size, PROT_READ | PROT_EXEC) == -1) goto fail;
// We need not `munmap(tmem, size)` in the above `OnScopeLeave` on success.

// We need not `munmap(tmem, size)` on success.
tmem = nullptr;
return 0;
goto done;
fail:
PrintSystemError(errno);
return -1;
status = -1;
done:
if (nmem != nullptr && nmem != MAP_FAILED && munmap(nmem, size) == -1)
PrintSystemError(errno);
if (tmem != nullptr && tmem != MAP_FAILED && munmap(tmem, size) == -1)
PrintSystemError(errno);
return status;
}
#endif // defined(NODE_ENABLE_LARGE_CODE_PAGES) && NODE_ENABLE_LARGE_CODE_PAGES

Expand Down

0 comments on commit c9b1139

Please sign in to comment.