diff --git a/.eslintrc.js b/.eslintrc.js index 55b61cea63be6c..32c85741802f39 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -8,6 +8,10 @@ const path = require('path'); const NodePlugin = require('./tools/node_modules/eslint-plugin-node-core'); NodePlugin.RULES_DIR = path.resolve(__dirname, 'tools', 'eslint-rules'); +// The Module._findPath() monkeypatching is to make it so that ESLint will work +// if invoked by a globally-installed ESLint or ESLint installed elsewhere +// rather than the one we ship. This makes it possible for IDEs to lint files +// with our rules while people edit them. const ModuleFindPath = Module._findPath; const hacks = [ 'eslint-plugin-node-core', @@ -236,7 +240,7 @@ module.exports = { { selector: "CallExpression[callee.property.name='strictEqual'][arguments.0.type='Literal']:not([arguments.1.type='Literal']):not([arguments.1.type='ObjectExpression']):not([arguments.1.type='ArrayExpression']):not([arguments.1.type='UnaryExpression'])", message: 'The first argument should be the `actual`, not the `expected` value.', - } + }, ], /* eslint-enable max-len */ 'no-return-await': 'error', diff --git a/.github/SUPPORT.md b/.github/SUPPORT.md new file mode 100644 index 00000000000000..1a086bd2cc59a7 --- /dev/null +++ b/.github/SUPPORT.md @@ -0,0 +1,26 @@ +# Support + +Node.js contributors have limited availability to address general support +questions. Please make sure you are using a [currently-supported version of +Node.js](https://github.com/nodejs/Release#release-schedule). + +When looking for support, please first search for your question in these venues: + +* [Node.js Website](https://nodejs.org/en/), especially the + [API docs](https://nodejs.org/api/) +* [Node.js Help](https://github.com/nodejs/help) +* [Open or closed issues in the Node.js GitHub organization](https://github.com/issues?utf8=%E2%9C%93&q=sort%3Aupdated-desc+org%3Anodejs+is%3Aissue) + +If you didn't find an answer in the resources above, try these unofficial +resources: + +* [Questions tagged 'node.js' on Stack Overflow](https://stackoverflow.com/questions/tagged/node.js) +* [#node.js channel on chat.freenode.net](https://webchat.freenode.net?channels=node.js&uio=d4) +* [Node.js Slack Community](https://node-js.slack.com/) + * To register: [nodeslackers.com](http://www.nodeslackers.com/) + +GitHub issues are for tracking enhancements and bugs, not general support. + +The open source license grants you the freedom to use Node.js. It does not +guarantee commitments of other people's time. Please be respectful and manage +your expectations. diff --git a/CHANGELOG.md b/CHANGELOG.md index b7bf473720886a..6d36375decdbd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,8 @@ release. -12.3.1
+12.4.0
+12.3.1
12.3.0
12.2.0
12.1.0
diff --git a/README.md b/README.md index 0ab0dfbd6d65cf..185aac7f455430 100644 --- a/README.md +++ b/README.md @@ -28,37 +28,16 @@ The Node.js project uses an [open governance model](./GOVERNANCE.md). The * [Verifying Binaries](#verifying-binaries) * [Building Node.js](#building-nodejs) * [Security](#security) +* [Contributing to Node.js](#contributing-to-nodejs) * [Current Project Team Members](#current-project-team-members) * [TSC (Technical Steering Committee)](#tsc-technical-steering-committee) * [Collaborators](#collaborators) * [Release Keys](#release-keys) -* [Contributing to Node.js](#contributing-to-nodejs) ## Support -Node.js contributors have limited availability to address general support -questions. Please make sure you are using a [currently-supported version of -Node.js](https://github.com/nodejs/Release#release-schedule). - -When looking for support, please first search for your question in these venues: - -* [Node.js Website][] -* [Node.js Help][] -* [Open or closed issues in the Node.js GitHub organization](https://github.com/issues?utf8=%E2%9C%93&q=sort%3Aupdated-desc+org%3Anodejs+is%3Aissue) - -If you didn't find an answer in the resources above, try these unofficial -resources: - -* [Questions tagged 'node.js' on StackOverflow][] -* [#node.js channel on chat.freenode.net][] -* [Node.js Slack Community](https://node-js.slack.com/) - * To register: [nodeslackers.com](http://www.nodeslackers.com/) - -GitHub issues are for tracking enhancements and bugs, not general support. - -The open source license grants you the freedom to use Node.js. It does not -guarantee commitments of other people's time. Please be respectful and manage -your expectations. +Looking for help? Check out the +[instructions for getting support](.github/SUPPORT.md). ## Release Types @@ -160,6 +139,12 @@ source and a list of supported platforms. For information on reporting security vulnerabilities in Node.js, see [SECURITY.md](./SECURITY.md). +## Contributing to Node.js + +* [Contributing to the project][] +* [Working Groups][] +* [Strategic Initiatives][] + ## Current Project Team Members For information about the governance of the Node.js project, see @@ -400,7 +385,7 @@ For information about the governance of the Node.js project, see * [Qard](https://github.com/Qard) - **Stephen Belanger** <admin@stephenbelanger.com> (he/him) * [refack](https://github.com/refack) - -**Refael Ackermann** <refack@gmail.com> (he/him) +**Refael Ackermann (רפאל פלחי)** <refack@gmail.com> (he/him/הוא/אתה) * [richardlau](https://github.com/richardlau) - **Richard Lau** <riclau@uk.ibm.com> * [ronkorving](https://github.com/ronkorving) - @@ -593,18 +578,8 @@ Other keys used to sign some previous releases: * **Timothy J Fontaine** <tjfontaine@gmail.com> `7937DFD2AB06298B2293C3187D33FF9D0246406D` -## Contributing to Node.js - -* [Contributing to the project][] -* [Working Groups][] -* [Strategic Initiatives][] - [Code of Conduct]: https://github.com/nodejs/admin/blob/master/CODE_OF_CONDUCT.md [Contributing to the project]: CONTRIBUTING.md -[Node.js Help]: https://github.com/nodejs/help [Node.js Foundation]: https://nodejs.org/en/foundation/ -[Node.js Website]: https://nodejs.org/en/ -[Questions tagged 'node.js' on StackOverflow]: https://stackoverflow.com/questions/tagged/node.js [Working Groups]: https://github.com/nodejs/TSC/blob/master/WORKING_GROUPS.md [Strategic Initiatives]: https://github.com/nodejs/TSC/blob/master/Strategic-Initiatives.md -[#node.js channel on chat.freenode.net]: https://webchat.freenode.net?channels=node.js&uio=d4 diff --git a/deps/histogram/histogram.gyp b/deps/histogram/histogram.gyp index bcfa198f9d37c1..e3f5fd7a46bb72 100644 --- a/deps/histogram/histogram.gyp +++ b/deps/histogram/histogram.gyp @@ -3,6 +3,10 @@ { 'target_name': 'histogram', 'type': 'static_library', + 'cflags': ['-fvisibility=hidden'], + 'xcode_settings': { + 'GCC_SYMBOLS_PRIVATE_EXTERN': 'YES', # -fvisibility=hidden + }, 'include_dirs': ['src'], 'direct_dependent_settings': { 'include_dirs': [ 'src' ] diff --git a/doc/api/assert.md b/doc/api/assert.md index 0a7d35f105e48e..22383a16b62d14 100644 --- a/doc/api/assert.md +++ b/doc/api/assert.md @@ -68,7 +68,7 @@ try { } catch (err) { assert(err instanceof assert.AssertionError); assert.strictEqual(err.message, message); - assert.strictEqual(err.name, 'AssertionError [ERR_ASSERTION]'); + assert.strictEqual(err.name, 'AssertionError'); assert.strictEqual(err.actual, 1); assert.strictEqual(err.expected, 2); assert.strictEqual(err.code, 'ERR_ASSERTION'); diff --git a/doc/api/cli.md b/doc/api/cli.md index 42950418282ef2..e6e507ddb6c5c5 100644 --- a/doc/api/cli.md +++ b/doc/api/cli.md @@ -148,13 +148,6 @@ the ability to import a directory that has an index file. Please see [customizing esm specifier resolution][] for example usage. -### `--experimental-json-modules` - - -Enable experimental JSON support for the ES Module loader. - ### `--experimental-modules` + +Enable experimental WebAssembly module support. + ### `--force-fips` +### `--heap-prof` + + +> Stability: 1 - Experimental + +Starts the V8 heap profiler on start up, and writes the heap profile to disk +before exit. + +If `--heap-prof-dir` is not specified, the generated profile will be placed +in the current working directory. + +If `--heap-prof-name` is not specified, the generated profile will be +named `Heap.${yyyymmdd}.${hhmmss}.${pid}.${tid}.${seq}.heapprofile`. + +```console +$ node --heap-prof index.js +$ ls *.heapprofile +Heap.20190409.202950.15293.0.001.heapprofile +``` + +### `--heap-prof-dir` + + +> Stability: 1 - Experimental + +Specify the directory where the heap profiles generated by `--heap-prof` will +be placed. + +### `--heap-prof-interval` + + +> Stability: 1 - Experimental + +Specify the average sampling interval in bytes for the heap profiles generated +by `--heap-prof`. The default is 512 * 1024 bytes. + +### `--heap-prof-name` + + +> Stability: 1 - Experimental + +Specify the file name of the heap profile generated by `--heap-prof`. + Generates a heap snapshot each time the process receives the specified signal. `signal` must be a valid signal name. Disabled by default. @@ -271,6 +322,17 @@ This flag exists to aid in experimentation with the internal implementation of the Node.js http parser. This flag is likely to become a no-op and removed at some point in the future. +### `--http-server-default-timeout=milliseconds` + + +Overrides the default value of `http`, `https` and `http2` server socket +timeout. Setting the value to 0 disables server socket timeout. Unless +provided, http server sockets timeout after 120s (2 minutes). Programmatic +setting of the timeout takes precedence over the value set through this +flag. + ### `--icu-data-dir=file` +Type: Runtime + The undocumented `net._setSimultaneousAccepts()` function was originally intended for debugging and performance tuning when using the `child_process` and `cluster` modules on Windows. The function is not generally useful and diff --git a/doc/api/documentation.md b/doc/api/documentation.md index 612b6f2dbf545b..c0733f266dd830 100644 --- a/doc/api/documentation.md +++ b/doc/api/documentation.md @@ -9,24 +9,17 @@ Node.js is a JavaScript runtime built on the [V8 JavaScript engine][]. ## Contributing -If errors are found in this documentation, please [submit an issue][] -or see [the contributing guide][] for directions on how to submit a patch. +Report errors in this documentation in [the issue tracker][]. See +[the contributing guide][] for directions on how to submit pull requests. -Every file is generated based on the corresponding `.md` file in the -`doc/api/` folder in Node.js's source tree. The documentation is generated -using the `tools/doc/generate.js` program. An HTML template is located at -`doc/template.html`. ## Stability Index -Throughout the documentation are indications of a section's -stability. The Node.js API is still somewhat changing, and as it -matures, certain parts are more reliable than others. Some are so -proven, and so relied upon, that they are unlikely to ever change at -all. Others are brand new and experimental, or known to be hazardous -and being redesigned. +Throughout the documentation are indications of a section's stability. Some APIs +are so proven and so relied upon that they are unlikely to ever change at all. +Others are brand new and experimental, or known to be hazardous. The stability indices are as follows: @@ -35,55 +28,38 @@ The stability indices are as follows: -> Stability: 1 - Experimental. This feature is still under active development -> and subject to non-backward compatible changes or removal in any future -> version. Use of the feature is not recommended in production environments. -> Experimental features are not subject to the Node.js Semantic Versioning -> model. +> Stability: 1 - Experimental. The feature is not subject to Semantic Versioning +> rules. Non-backward compatible changes or removal may occur in any future +> release. Use of the feature is not recommended in production environments. > Stability: 2 - Stable. Compatibility with the npm ecosystem is a high > priority. -Use caution when making use of `Experimental` features, particularly -within modules that are dependencies (or dependencies of -dependencies) within a Node.js application. End users may not be aware that -experimental features are being used, and may experience unexpected -failures or behavior changes when API modifications occur. To help avoid such -surprises, `Experimental` features may require a command-line flag to -enable them, or may emit a process warning. -By default, such warnings are printed to [`stderr`][] and may be handled by -attaching a listener to the [`'warning'`][] event. +Use caution when making use of Experimental features, particularly within +modules. End users may not be aware that experimental features are being used. +Bugs or behavior changes may surprise end users when Experimental API +modifications occur. To avoid surprises, use of an Experimental feature may need +a command-line flag. Experimental features may also emit a [warning][]. ## JSON Output -> Stability: 1 - Experimental +Every `.html` document has a corresponding `.json` document. This is for IDEs +and other utilities that consume the documentation. -Every `.html` document has a corresponding `.json` document presenting -the same information in a structured manner. This feature is -experimental, and added for the benefit of IDEs and other utilities that -wish to do programmatic things with the documentation. +## System calls and man pages -## Syscalls and man pages +Node.js functions which wrap a system call will document that. The docs link +to the corresponding man pages which describe how the system call works. -System calls like open(2) and read(2) define the interface between user programs -and the underlying operating system. Node.js functions -which wrap a syscall, -like [`fs.open()`][], will document that. The docs link to the corresponding man -pages (short for manual pages) which describe how the syscalls work. +Most Unix system calls have Windows analogues. Still, behavior differences may +be unavoidable. -Most Unix syscalls have Windows equivalents, but behavior may differ on Windows -relative to Linux and macOS. For an example of the subtle ways in which it's -sometimes impossible to replace Unix syscall semantics on Windows, see [Node.js -issue 4760](https://github.com/nodejs/node/issues/4760). - -[`'warning'`]: process.html#process_event_warning -[`fs.open()`]: fs.html#fs_fs_open_path_flags_mode_callback -[`stderr`]: process.html#process_process_stderr -[submit an issue]: https://github.com/nodejs/node/issues/new [the contributing guide]: https://github.com/nodejs/node/blob/master/CONTRIBUTING.md +[the issue tracker]: https://github.com/nodejs/node/issues/new [V8 JavaScript engine]: https://v8.dev/ +[warning]: process.html#process_event_warning diff --git a/doc/api/errors.md b/doc/api/errors.md index 9420e07b16c871..ebbe215b6075f1 100644 --- a/doc/api/errors.md +++ b/doc/api/errors.md @@ -2349,7 +2349,7 @@ such as `process.stdout.on('data')`. [`sign.sign()`]: crypto.html#crypto_sign_sign_privatekey_outputencoding [`stream.pipe()`]: stream.html#stream_readable_pipe_destination_options [`stream.push()`]: stream.html#stream_readable_push_chunk_encoding -[`stream.unshift()`]: stream.html#stream_readable_unshift_chunk +[`stream.unshift()`]: stream.html#stream_readable_unshift_chunk_encoding [`stream.write()`]: stream.html#stream_writable_write_chunk_encoding_callback [`subprocess.kill()`]: child_process.html#child_process_subprocess_kill_signal [`subprocess.send()`]: child_process.html#child_process_subprocess_send_message_sendhandle_options_callback diff --git a/doc/api/esm.md b/doc/api/esm.md index 646e7d5e9966ec..1ea41e84c1d53d 100644 --- a/doc/api/esm.md +++ b/doc/api/esm.md @@ -409,18 +409,11 @@ fs.readFileSync = () => Buffer.from('Hello, ESM'); fs.readFileSync === readFileSync; ``` -## Experimental JSON Modules +## JSON Modules -**Note: This API is still being designed and is subject to change.** +JSON modules follow the [WHATWG JSON modules specification][]. -Currently importing JSON modules are only supported in the `commonjs` mode -and are loaded using the CJS loader. [WHATWG JSON modules][] are currently -being standardized, and are experimentally supported by including the -additional flag `--experimental-json-modules` when running Node.js. - -When the `--experimental-json-modules` flag is included both the -`commonjs` and `module` mode will use the new experimental JSON -loader. The imported JSON only exposes a `default`, there is no +The imported JSON only exposes a `default`. There is no support for named exports. A cache entry is created in the CommonJS cache, to avoid duplication. The same object will be returned in CommonJS if the JSON module has already been imported from the @@ -433,14 +426,6 @@ Assuming an `index.mjs` with import packageConfig from './package.json'; ``` -The `--experimental-json-modules` flag is needed for the module -to work. - -```bash -node --experimental-modules index.mjs # fails -node --experimental-modules --experimental-json-modules index.mjs # works -``` - ## Experimental Wasm Modules Importing Web Assembly modules is supported under the @@ -763,7 +748,7 @@ success! [CommonJS]: modules.html [ECMAScript-modules implementation]: https://github.com/nodejs/modules/blob/master/doc/plan-for-new-modules-implementation.md [Node.js EP for ES Modules]: https://github.com/nodejs/node-eps/blob/master/002-es-modules.md -[WHATWG JSON modules]: https://github.com/whatwg/html/issues/4315 +[WHATWG JSON modules specification]: https://html.spec.whatwg.org/#creating-a-json-module-script [ES Module Integration Proposal for Web Assembly]: https://github.com/webassembly/esm-integration [dynamic instantiate hook]: #esm_dynamic_instantiate_hook [the official standard format]: https://tc39.github.io/ecma262/#sec-modules diff --git a/doc/api/fs.md b/doc/api/fs.md index 3b3c6efd58243f..d2506d28e33cce 100644 --- a/doc/api/fs.md +++ b/doc/api/fs.md @@ -2898,7 +2898,8 @@ changes: Asynchronously rename file at `oldPath` to the pathname provided as `newPath`. In the case that `newPath` already exists, it will -be overwritten. No arguments other than a possible exception are +be overwritten. If there is a directory at `newPath`, an error will +be raised instead. No arguments other than a possible exception are given to the completion callback. See also: rename(2). @@ -3733,9 +3734,14 @@ added: v10.0.0 A `FileHandle` object is a wrapper for a numeric file descriptor. Instances of `FileHandle` are distinct from numeric file descriptors -in that, if the `FileHandle` is not explicitly closed using the -`filehandle.close()` method, they will automatically close the file descriptor +in that they provide an object oriented API for working with files. + +If a `FileHandle` is not closed using the +`filehandle.close()` method, it might automatically close the file descriptor and will emit a process warning, thereby helping to prevent memory leaks. +Please do not rely on this behavior in your code because it is unreliable and +your file may not be closed. Instead, always explicitly close `FileHandle`s. +Node.js may change this behavior in the future. Instances of the `FileHandle` object are created internally by the `fsPromises.open()` method. diff --git a/doc/api/http.md b/doc/api/http.md index ccd5e4b0bd4b30..da0de7c1e68603 100644 --- a/doc/api/http.md +++ b/doc/api/http.md @@ -1030,6 +1030,9 @@ By default, the Server's timeout value is 2 minutes, and sockets are destroyed automatically if they time out. However, if a callback is assigned to the Server's `'timeout'` event, timeouts must be handled explicitly. +To change the default timeout use the [`--http-server-default-timeout`][] +flag. + ### server.timeout It is convenient to organize programs and libraries into self-contained -directories, and then provide a single entry point to that library. +directories, and then provide a single entry point to those directories. There are three ways in which a folder may be passed to `require()` as an argument. diff --git a/doc/api/n-api.md b/doc/api/n-api.md index a39f62aa74cc2b..f648ab1d18f752 100644 --- a/doc/api/n-api.md +++ b/doc/api/n-api.md @@ -7,7 +7,7 @@ N-API (pronounced N as in the letter, followed by API) is an API for building native Addons. It is independent from -the underlying JavaScript runtime (ex V8) and is maintained as part of +the underlying JavaScript runtime (for example, V8) and is maintained as part of Node.js itself. This API will be Application Binary Interface (ABI) stable across versions of Node.js. It is intended to insulate Addons from changes in the underlying JavaScript engine and allow modules @@ -148,7 +148,7 @@ available to the module code. | | 1 | 2 | 3 | 4 | |:-----:|:-------:|:--------:|:--------:|:--------:| | v6.x | | | v6.14.2* | | -| v8.x | v8.0.0* | v8.10.0* | v8.11.2 | | +| v8.x | v8.0.0* | v8.10.0* | v8.11.2 | v8.16.0 | | v9.x | v9.0.0* | v9.3.0* | v9.11.0* | | | v10.x | | | v10.0.0 | | | v11.x | | | v11.0.0 | v11.8.0 | @@ -222,6 +222,10 @@ consumed by the various APIs. These APIs should be treated as opaque, introspectable only with other N-API calls. ### napi_status + Integral status code indicating the success or failure of a N-API call. Currently, the following status codes are supported. ```C @@ -251,6 +255,10 @@ If additional information is required upon an API returning a failed status, it can be obtained by calling `napi_get_last_error_info`. ### napi_extended_error_info + ```C typedef struct { const char* error_message; @@ -283,12 +291,20 @@ not allowed. This is an opaque pointer that is used to represent a JavaScript value. ### napi_threadsafe_function + This is an opaque pointer that represents a JavaScript function which can be called asynchronously from multiple threads via `napi_call_threadsafe_function()`. ### napi_threadsafe_function_release_mode + A value to be given to `napi_release_threadsafe_function()` to indicate whether the thread-safe function is to be closed immediately (`napi_tsfn_abort`) or @@ -302,6 +318,10 @@ typedef enum { ``` ### napi_threadsafe_function_call_mode + A value to be given to `napi_call_threadsafe_function()` to indicate whether the call should block whenever the queue associated with the thread-safe @@ -333,10 +353,18 @@ longer referenced from the current stack frame. For more details, review the [Object Lifetime Management][]. #### napi_escapable_handle_scope + Escapable handle scopes are a special type of handle scope to return values created within a particular handle scope to a parent scope. #### napi_ref + This is the abstraction to use to reference a `napi_value`. This allows for users to manage the lifetimes of JavaScript values, including defining their minimum lifetimes explicitly. @@ -345,11 +373,19 @@ For more details, review the [Object Lifetime Management][]. ### N-API Callback types #### napi_callback_info + Opaque datatype that is passed to a callback function. It can be used for getting additional information about the context in which the callback was invoked. #### napi_callback + Function pointer type for user-provided native functions which are to be exposed to JavaScript via N-API. Callback functions should satisfy the following signature: @@ -358,6 +394,10 @@ typedef napi_value (*napi_callback)(napi_env, napi_callback_info); ``` #### napi_finalize + Function pointer type for add-on provided functions that allow the user to be notified when externally-owned data is ready to be cleaned up because the object with which it was associated with, has been garbage-collected. The user @@ -372,6 +412,10 @@ typedef void (*napi_finalize)(napi_env env, ``` #### napi_async_execute_callback + Function pointer used with functions that support asynchronous operations. Callback functions must satisfy the following signature: @@ -385,6 +429,10 @@ JavaScript objects. Most often, any code that needs to make N-API calls should be made in `napi_async_complete_callback` instead. #### napi_async_complete_callback + Function pointer used with functions that support asynchronous operations. Callback functions must satisfy the following signature: @@ -395,6 +443,10 @@ typedef void (*napi_async_complete_callback)(napi_env env, ``` #### napi_threadsafe_function_call_js + Function pointer used with asynchronous thread-safe function calls. The callback will be called on the main thread. Its purpose is to use a data item arriving @@ -460,6 +512,10 @@ In order to retrieve this information [`napi_get_last_error_info`][] is provided which returns a `napi_extended_error_info` structure. The format of the `napi_extended_error_info` structure is as follows: + ```C typedef struct napi_extended_error_info { const char* error_message; diff --git a/doc/api/stream.md b/doc/api/stream.md index 8b525b953ba2bb..46f4226039c2df 100644 --- a/doc/api/stream.md +++ b/doc/api/stream.md @@ -1195,7 +1195,7 @@ setTimeout(() => { }, 1000); ``` -##### readable.unshift(chunk) +##### readable.unshift(chunk[, encoding]) ```js { @@ -149,7 +160,9 @@ to get swapped out by the operating system. heap_size_limit: 1535115264, malloced_memory: 16384, peak_malloced_memory: 1127496, - does_zap_garbage: 0 + does_zap_garbage: 0, + number_of_native_contexts: 1, + number_of_detached_contexts: 0 } ``` diff --git a/doc/changelogs/CHANGELOG_V12.md b/doc/changelogs/CHANGELOG_V12.md index 9eb0f5e401fcaa..7bd3ffd62fc391 100644 --- a/doc/changelogs/CHANGELOG_V12.md +++ b/doc/changelogs/CHANGELOG_V12.md @@ -9,6 +9,7 @@ +12.4.0
12.3.1
12.3.0
12.2.0
@@ -32,6 +33,142 @@ * [io.js](CHANGELOG_IOJS.md) * [Archive](CHANGELOG_ARCHIVE.md) + +## 2019-06-04, Version 12.4.0 (Current), @targos + +### Notable changes + +* **doc**: + * The JSON variant of the API documentation is no longer experimental (Rich Trott) [#27842](https://github.com/nodejs/node/pull/27842). +* **esm**: + * JSON module support is always enabled under `--experimental-modules`. The + `--experimental-json-modules` flag has been removed (Myles Borins) [#27752](https://github.com/nodejs/node/pull/27752). +* **http,http2**: + * A new flag has been added for overriding the default HTTP server socket + timeout (which is two minutes). Pass `--http-server-default-timeout=milliseconds` + or `--http-server-default-timeout=0` to respectively change or disable the timeout. + Starting with Node.js 13.0.0, the timeout will be disabled by default (Ali Ijaz Sheikh) [#27704](https://github.com/nodejs/node/pull/27704). +* **inspector**: + * Added an experimental `--heap-prof` flag to start the V8 heap profiler + on startup and write the heap profile to disk before exit (Joyee Cheung) [#27596](https://github.com/nodejs/node/pull/27596). +* **stream**: + * The `readable.unshift()` method now correctly converts strings to buffers. + Additionally, a new optional argument is accepted to specify the string's + encoding, such as `'utf8'` or `'ascii'` (Marcos Casagrande) [#27194](https://github.com/nodejs/node/pull/27194). +* **v8**: + * The object returned by `v8.getHeapStatistics()` has two new properties: + `number_of_native_contexts` and `number_of_detached_contexts` (Yuriy Vasiyarov) [#27933](https://github.com/nodejs/node/pull/27933). + +### Commits + +* [[`5bbc6d79c3`](https://github.com/nodejs/node/commit/5bbc6d79c3)] - **assert**: remove unreachable code (Rich Trott) [#27840](https://github.com/nodejs/node/pull/27840) +* [[`530e63a4eb`](https://github.com/nodejs/node/commit/530e63a4eb)] - **assert**: remove unreachable code (Rich Trott) [#27786](https://github.com/nodejs/node/pull/27786) +* [[`9b08c458be`](https://github.com/nodejs/node/commit/9b08c458be)] - **build,aix**: link with `noerrmsg` to eliminate warnings (Refael Ackermann) [#27773](https://github.com/nodejs/node/pull/27773) +* [[`08b0ca9645`](https://github.com/nodejs/node/commit/08b0ca9645)] - **build,win**: create junction instead of symlink to `out\\%config%` (Refael Ackermann) [#27736](https://github.com/nodejs/node/pull/27736) +* [[`ea2d550507`](https://github.com/nodejs/node/commit/ea2d550507)] - **child_process**: move exports to bottom for consistent code style (himself65) [#27845](https://github.com/nodejs/node/pull/27845) +* [[`a9f95572c3`](https://github.com/nodejs/node/commit/a9f95572c3)] - **child_process**: remove extra shallow copy (zero1five) [#27801](https://github.com/nodejs/node/pull/27801) +* [[`449ee8dd42`](https://github.com/nodejs/node/commit/449ee8dd42)] - **console**: fix table() output (Brian White) [#27917](https://github.com/nodejs/node/pull/27917) +* [[`9220a68a76`](https://github.com/nodejs/node/commit/9220a68a76)] - **crypto**: fix KeyObject handle type error message (Alexander Avakov) [#27904](https://github.com/nodejs/node/pull/27904) +* [[`3b6424fa29`](https://github.com/nodejs/node/commit/3b6424fa29)] - **deps**: histogram: unexport symbols (Ben Noordhuis) [#27779](https://github.com/nodejs/node/pull/27779) +* [[`ef25ac5223`](https://github.com/nodejs/node/commit/ef25ac5223)] - **doc**: clarify wording in modules.md (Alex Temny) [#27853](https://github.com/nodejs/node/pull/27853) +* [[`c683cd99d7`](https://github.com/nodejs/node/commit/c683cd99d7)] - **doc**: improve explanation for directory with fs.rename() (Rich Trott) [#27963](https://github.com/nodejs/node/pull/27963) +* [[`70b485478c`](https://github.com/nodejs/node/commit/70b485478c)] - **doc**: fix the wrong name of AssertionError (Kyle Zhang) [#27982](https://github.com/nodejs/node/pull/27982) +* [[`11c3ddb4cb`](https://github.com/nodejs/node/commit/11c3ddb4cb)] - **doc**: simplify system call material in doc overview (Rich Trott) [#27966](https://github.com/nodejs/node/pull/27966) +* [[`c56640138a`](https://github.com/nodejs/node/commit/c56640138a)] - **doc**: warn about relying on fs gc close behavior (Benjamin Gruenbaum) [#27972](https://github.com/nodejs/node/pull/27972) +* [[`bab9f5a891`](https://github.com/nodejs/node/commit/bab9f5a891)] - **doc**: add information to revoked deprecations (cjihrig) [#27952](https://github.com/nodejs/node/pull/27952) +* [[`f4fc75d245`](https://github.com/nodejs/node/commit/f4fc75d245)] - **doc**: add missing status to DEP0121 (cjihrig) [#27950](https://github.com/nodejs/node/pull/27950) +* [[`77ff597faa`](https://github.com/nodejs/node/commit/77ff597faa)] - **doc**: add missing --experimental-wasm-modules docs (cjihrig) [#27948](https://github.com/nodejs/node/pull/27948) +* [[`6ca4f03ccf`](https://github.com/nodejs/node/commit/6ca4f03ccf)] - **doc**: revise additional Experimental status text (Rich Trott) [#27931](https://github.com/nodejs/node/pull/27931) +* [[`a1788de0a4`](https://github.com/nodejs/node/commit/a1788de0a4)] - **doc**: adds link to nightly code coverage report (Tariq Ramlall) [#27922](https://github.com/nodejs/node/pull/27922) +* [[`b7cd0de145`](https://github.com/nodejs/node/commit/b7cd0de145)] - **doc**: fix typo in pipe from async iterator example (Luigi Pinca) [#27870](https://github.com/nodejs/node/pull/27870) +* [[`f621b8f178`](https://github.com/nodejs/node/commit/f621b8f178)] - **doc**: reword Experimental stability index (Rich Trott) [#27879](https://github.com/nodejs/node/pull/27879) +* [[`7a7fc4e7e6`](https://github.com/nodejs/node/commit/7a7fc4e7e6)] - **doc**: update n-api support matrix (teams2ua) [#27567](https://github.com/nodejs/node/pull/27567) +* [[`9d9b32eff5`](https://github.com/nodejs/node/commit/9d9b32eff5)] - **doc**: fix for OutgoingMessage.prototype.\_headers/\_headerNames (Daniel Nalborczyk) [#27574](https://github.com/nodejs/node/pull/27574) +* [[`263e53317b`](https://github.com/nodejs/node/commit/263e53317b)] - **doc**: reposition "How to Contribute" README section (Anish Asrani) [#27811](https://github.com/nodejs/node/pull/27811) +* [[`85f505c292`](https://github.com/nodejs/node/commit/85f505c292)] - **doc**: add version info for types (Michael Dawson) [#27754](https://github.com/nodejs/node/pull/27754) +* [[`e3bb2aef60`](https://github.com/nodejs/node/commit/e3bb2aef60)] - **doc**: remove experimental status for JSON documentation (Rich Trott) [#27842](https://github.com/nodejs/node/pull/27842) +* [[`6981565c20`](https://github.com/nodejs/node/commit/6981565c20)] - **doc**: edit stability index overview (Rich Trott) [#27831](https://github.com/nodejs/node/pull/27831) +* [[`1a8e67cc1f`](https://github.com/nodejs/node/commit/1a8e67cc1f)] - **doc**: simplify contributing documentation (Rich Trott) [#27785](https://github.com/nodejs/node/pull/27785) +* [[`041b2220be`](https://github.com/nodejs/node/commit/041b2220be)] - **doc,n-api**: fix typo in N-API introduction (Richard Lau) [#27833](https://github.com/nodejs/node/pull/27833) +* [[`6cd64c8279`](https://github.com/nodejs/node/commit/6cd64c8279)] - **doc,test**: clarify that Http2Stream is destroyed after data is read (Alba Mendez) [#27891](https://github.com/nodejs/node/pull/27891) +* [[`cc69d5af8e`](https://github.com/nodejs/node/commit/cc69d5af8e)] - **doc,tools**: get altDocs versions from CHANGELOG.md (Richard Lau) [#27661](https://github.com/nodejs/node/pull/27661) +* [[`e72d4aa522`](https://github.com/nodejs/node/commit/e72d4aa522)] - **errors**: create internal connResetException (Rich Trott) [#27953](https://github.com/nodejs/node/pull/27953) +* [[`be1166fd01`](https://github.com/nodejs/node/commit/be1166fd01)] - **esm**: refactor createDynamicModule() (cjihrig) [#27809](https://github.com/nodejs/node/pull/27809) +* [[`e66648e887`](https://github.com/nodejs/node/commit/e66648e887)] - **(SEMVER-MINOR)** **esm**: remove experimental status from JSON modules (Myles Borins) [#27752](https://github.com/nodejs/node/pull/27752) +* [[`d948656635`](https://github.com/nodejs/node/commit/d948656635)] - **http**: fix deferToConnect comments (Robert Nagy) [#27876](https://github.com/nodejs/node/pull/27876) +* [[`24eaeed393`](https://github.com/nodejs/node/commit/24eaeed393)] - **http**: fix socketOnWrap edge cases (Anatoli Papirovski) [#27968](https://github.com/nodejs/node/pull/27968) +* [[`8b38dfbf39`](https://github.com/nodejs/node/commit/8b38dfbf39)] - **http**: call write callback even if there is no message body (Luigi Pinca) [#27777](https://github.com/nodejs/node/pull/27777) +* [[`588fd0c20d`](https://github.com/nodejs/node/commit/588fd0c20d)] - **(SEMVER-MINOR)** **http, http2**: flag for overriding server timeout (Ali Ijaz Sheikh) [#27704](https://github.com/nodejs/node/pull/27704) +* [[`799aeca134`](https://github.com/nodejs/node/commit/799aeca134)] - **http2**: respect inspect() depth (cjihrig) [#27983](https://github.com/nodejs/node/pull/27983) +* [[`83aaef87d0`](https://github.com/nodejs/node/commit/83aaef87d0)] - **http2**: fix tracking received data for maxSessionMemory (Anna Henningsen) [#27914](https://github.com/nodejs/node/pull/27914) +* [[`8c35198499`](https://github.com/nodejs/node/commit/8c35198499)] - **http2**: support net.Server options (Luigi Pinca) [#27782](https://github.com/nodejs/node/pull/27782) +* [[`23119cacf8`](https://github.com/nodejs/node/commit/23119cacf8)] - **inspector**: supported NodeRuntime domain in worker (Aleksei Koziatinskii) [#27706](https://github.com/nodejs/node/pull/27706) +* [[`89483be254`](https://github.com/nodejs/node/commit/89483be254)] - **inspector**: more conservative minimum stack size (Ben Noordhuis) [#27855](https://github.com/nodejs/node/pull/27855) +* [[`512ab1fddf`](https://github.com/nodejs/node/commit/512ab1fddf)] - **inspector**: removing checking of non existent field in lib/inspector.js (Keroosha) [#27919](https://github.com/nodejs/node/pull/27919) +* [[`d99e70381e`](https://github.com/nodejs/node/commit/d99e70381e)] - **SEMVER-MINOR** **inspector**: implement --heap-prof (Joyee Cheung) [#27596](https://github.com/nodejs/node/pull/27596) +* [[`25eb05a97a`](https://github.com/nodejs/node/commit/25eb05a97a)] - **lib**: removed unnecessary fs.realpath `options` arg check + tests (Alex Pry) [#27909](https://github.com/nodejs/node/pull/27909) +* [[`9b90385825`](https://github.com/nodejs/node/commit/9b90385825)] - ***Revert*** "**lib**: print to stdout/stderr directly instead of using console" (Richard Lau) [#27823](https://github.com/nodejs/node/pull/27823) +* [[`18650579e8`](https://github.com/nodejs/node/commit/18650579e8)] - **meta**: correct personal info (Refael Ackermann (רפאל פלחי)) [#27940](https://github.com/nodejs/node/pull/27940) +* [[`d982f0b7e2`](https://github.com/nodejs/node/commit/d982f0b7e2)] - **meta**: create github support file (Gus Caplan) [#27926](https://github.com/nodejs/node/pull/27926) +* [[`2b7ad122b2`](https://github.com/nodejs/node/commit/2b7ad122b2)] - **n-api**: DRY napi\_coerce\_to\_x() API methods (Ben Noordhuis) [#27796](https://github.com/nodejs/node/pull/27796) +* [[`1da5acbf91`](https://github.com/nodejs/node/commit/1da5acbf91)] - **os**: assume UTF-8 for hostname (Anna Henningsen) [#27849](https://github.com/nodejs/node/pull/27849) +* [[`d406785814`](https://github.com/nodejs/node/commit/d406785814)] - **src**: unimplement deprecated v8-platform methods (Michaël Zasso) [#27872](https://github.com/nodejs/node/pull/27872) +* [[`33236b7c54`](https://github.com/nodejs/node/commit/33236b7c54)] - **(SEMVER-MINOR)** **src**: export number\_of\_native\_contexts and number\_of\_detached\_contexts (Yuriy Vasiyarov) [#27933](https://github.com/nodejs/node/pull/27933) +* [[`1a179e1736`](https://github.com/nodejs/node/commit/1a179e1736)] - **src**: use ArrayBufferViewContents more frequently (Anna Henningsen) [#27920](https://github.com/nodejs/node/pull/27920) +* [[`b9cc4072e6`](https://github.com/nodejs/node/commit/b9cc4072e6)] - **src**: make UNREACHABLE variadic (Refael Ackermann) [#27877](https://github.com/nodejs/node/pull/27877) +* [[`44846aebd2`](https://github.com/nodejs/node/commit/44846aebd2)] - **src**: move DiagnosticFilename inlines into a -inl.h (Sam Roberts) [#27839](https://github.com/nodejs/node/pull/27839) +* [[`d774ea5cce`](https://github.com/nodejs/node/commit/d774ea5cce)] - **src**: remove env-inl.h from header files (Sam Roberts) [#27755](https://github.com/nodejs/node/pull/27755) +* [[`02f794a53f`](https://github.com/nodejs/node/commit/02f794a53f)] - **src**: remove memory\_tracker-inl.h from header files (Sam Roberts) [#27755](https://github.com/nodejs/node/pull/27755) +* [[`940577bd76`](https://github.com/nodejs/node/commit/940577bd76)] - **src**: move ThreadPoolWork inlines into a -inl.h (Sam Roberts) [#27755](https://github.com/nodejs/node/pull/27755) +* [[`c0cf17388c`](https://github.com/nodejs/node/commit/c0cf17388c)] - **src**: ignore SIGXFSZ, don't terminate (ulimit -f) (Ben Noordhuis) [#27798](https://github.com/nodejs/node/pull/27798) +* [[`a47ee80114`](https://github.com/nodejs/node/commit/a47ee80114)] - **(SEMVER-MINOR)** **stream**: convert string to Buffer when calling `unshift(\)` (Marcos Casagrande) [#27194](https://github.com/nodejs/node/pull/27194) +* [[`5eccd642ef`](https://github.com/nodejs/node/commit/5eccd642ef)] - **stream**: convert existing buffer when calling .setEncoding (Anna Henningsen) [#27936](https://github.com/nodejs/node/pull/27936) +* [[`6a5ce36fb8`](https://github.com/nodejs/node/commit/6a5ce36fb8)] - **test**: handle unknown message type in worker threads (Rich Trott) [#27995](https://github.com/nodejs/node/pull/27995) +* [[`182725651b`](https://github.com/nodejs/node/commit/182725651b)] - **test**: add coverage for unserializable worker thread error (Rich Trott) [#27995](https://github.com/nodejs/node/pull/27995) +* [[`887dd604f1`](https://github.com/nodejs/node/commit/887dd604f1)] - **test**: simplify fs promises test (Daniel Nalborczyk) [#27242](https://github.com/nodejs/node/pull/27242) +* [[`9229825496`](https://github.com/nodejs/node/commit/9229825496)] - **test**: covering destroying when worker already disconnected (Keroosha) [#27896](https://github.com/nodejs/node/pull/27896) +* [[`10bdd13972`](https://github.com/nodejs/node/commit/10bdd13972)] - **test**: rename test-performance to test-perf-hooks (Ujjwal Sharma) [#27969](https://github.com/nodejs/node/pull/27969) +* [[`6129376cd9`](https://github.com/nodejs/node/commit/6129376cd9)] - **test**: add coverage for sparse array maxArrayLength (went.out) [#27901](https://github.com/nodejs/node/pull/27901) +* [[`38e3827ca8`](https://github.com/nodejs/node/commit/38e3827ca8)] - **test**: add util inspect null getter test (Mikhail Kuklin) [#27884](https://github.com/nodejs/node/pull/27884) +* [[`0e1ce2055e`](https://github.com/nodejs/node/commit/0e1ce2055e)] - **test**: rsa-pss generateKeyPairSync invalid option hash (Evgenii Shchepotev) [#27883](https://github.com/nodejs/node/pull/27883) +* [[`0d74198123`](https://github.com/nodejs/node/commit/0d74198123)] - **test**: cover import of a \*.node file with a policy manifest (Evgenii Shchepotev) [#27903](https://github.com/nodejs/node/pull/27903) +* [[`6f9aa3f722`](https://github.com/nodejs/node/commit/6f9aa3f722)] - **test**: add test cases for paramEncoding 'explicit' (oksana) [#27900](https://github.com/nodejs/node/pull/27900) +* [[`682319f449`](https://github.com/nodejs/node/commit/682319f449)] - **test**: switch assertEqual arguments (Evgenii Shchepotev) [#27910](https://github.com/nodejs/node/pull/27910) +* [[`b5b234deff`](https://github.com/nodejs/node/commit/b5b234deff)] - **test**: add testcase for SourceTextModule custom inspect (Grigory Gorshkov) [#27889](https://github.com/nodejs/node/pull/27889) +* [[`630cc3ac30`](https://github.com/nodejs/node/commit/630cc3ac30)] - **test**: cover util.inspect on boxed primitive with colors (Alexander Avakov) [#27897](https://github.com/nodejs/node/pull/27897) +* [[`67b692bdb9`](https://github.com/nodejs/node/commit/67b692bdb9)] - **test**: add test case for checking typeof mgf1Hash (Levin Eugene) [#27892](https://github.com/nodejs/node/pull/27892) +* [[`2a509d40f4`](https://github.com/nodejs/node/commit/2a509d40f4)] - **test**: switch assertEqual arguments (Evgenii Shchepotev) [#27912](https://github.com/nodejs/node/pull/27912) +* [[`3ba354aaaa`](https://github.com/nodejs/node/commit/3ba354aaaa)] - **test**: add test for util.inspect (Levin Eugene) [#27906](https://github.com/nodejs/node/pull/27906) +* [[`313077ea62`](https://github.com/nodejs/node/commit/313077ea62)] - **test**: expect wpt/encoding/encodeInto.any.js to fail (Joyee Cheung) [#27860](https://github.com/nodejs/node/pull/27860) +* [[`8fc6914d09`](https://github.com/nodejs/node/commit/8fc6914d09)] - **test**: update wpt/encoding to 7287608f90 (Joyee Cheung) [#27860](https://github.com/nodejs/node/pull/27860) +* [[`0f86c2b185`](https://github.com/nodejs/node/commit/0f86c2b185)] - **test**: run WPT in subdirectories (Joyee Cheung) [#27860](https://github.com/nodejs/node/pull/27860) +* [[`51ccdae445`](https://github.com/nodejs/node/commit/51ccdae445)] - **test**: expect wpt/encoding/streams to fail (Joyee Cheung) [#27860](https://github.com/nodejs/node/pull/27860) +* [[`652cadba1c`](https://github.com/nodejs/node/commit/652cadba1c)] - **test**: fix arguments order of comparsion functions (martyns0n) [#27907](https://github.com/nodejs/node/pull/27907) +* [[`b117f6d5d8`](https://github.com/nodejs/node/commit/b117f6d5d8)] - **test**: switch assertEqual arguments (Evgenii Shchepotev) [#27913](https://github.com/nodejs/node/pull/27913) +* [[`e7966bcb80`](https://github.com/nodejs/node/commit/e7966bcb80)] - **test**: unhardcode server port (MurkyMeow) [#27908](https://github.com/nodejs/node/pull/27908) +* [[`b83571d236`](https://github.com/nodejs/node/commit/b83571d236)] - **test**: add a test case for the path.posix.resolve (Grigorii K. Shartsev) [#27905](https://github.com/nodejs/node/pull/27905) +* [[`f5bb1b380f`](https://github.com/nodejs/node/commit/f5bb1b380f)] - **test**: switch actual value argument and expected in deepStrictEqual call (Kopachyov Vitaliy) [#27888](https://github.com/nodejs/node/pull/27888) +* [[`531669b917`](https://github.com/nodejs/node/commit/531669b917)] - **test**: fix test-http2-multiheaders-raw (Grigorii K. Shartsev) [#27885](https://github.com/nodejs/node/pull/27885) +* [[`724d9c89bc`](https://github.com/nodejs/node/commit/724d9c89bc)] - **test**: change expected and actual values in assert call (oksana) [#27881](https://github.com/nodejs/node/pull/27881) +* [[`34ef9e4a2b`](https://github.com/nodejs/node/commit/34ef9e4a2b)] - **test**: detect missing postmortem metadata (cjihrig) [#27828](https://github.com/nodejs/node/pull/27828) +* [[`bfcbab4c0c`](https://github.com/nodejs/node/commit/bfcbab4c0c)] - **test**: fix test-https-agent-additional-options (Rich Trott) [#27830](https://github.com/nodejs/node/pull/27830) +* [[`a4c1fd5ffc`](https://github.com/nodejs/node/commit/a4c1fd5ffc)] - **test**: refactor test-https-agent-additional-options (Rich Trott) [#27830](https://github.com/nodejs/node/pull/27830) +* [[`17abc8c942`](https://github.com/nodejs/node/commit/17abc8c942)] - **test**: favor arrow functions for anonymous callbacks (Rich Trott) [#27830](https://github.com/nodejs/node/pull/27830) +* [[`155b947251`](https://github.com/nodejs/node/commit/155b947251)] - **test**: replace flag with option (Rich Trott) [#27830](https://github.com/nodejs/node/pull/27830) +* [[`144db48b6d`](https://github.com/nodejs/node/commit/144db48b6d)] - **test**: update wpt/url to 418f7fabeb (Joyee Cheung) [#27822](https://github.com/nodejs/node/pull/27822) +* [[`65d4f734e0`](https://github.com/nodejs/node/commit/65d4f734e0)] - **test**: use ShellTestEnvironment in WPT (Joyee Cheung) [#27822](https://github.com/nodejs/node/pull/27822) +* [[`a9a400e604`](https://github.com/nodejs/node/commit/a9a400e604)] - **test**: update wpt/resources to e1fddfbf80 (Joyee Cheung) [#27822](https://github.com/nodejs/node/pull/27822) +* [[`8040d8b321`](https://github.com/nodejs/node/commit/8040d8b321)] - **test**: increase debugging information on failure (Rich Trott) [#27790](https://github.com/nodejs/node/pull/27790) +* [[`6548b91835`](https://github.com/nodejs/node/commit/6548b91835)] - **tls**: trace errors can show up as SSL errors (Sam Roberts) [#27841](https://github.com/nodejs/node/pull/27841) +* [[`0fe16edfab`](https://github.com/nodejs/node/commit/0fe16edfab)] - **tls**: group chunks into TLS segments (Alba Mendez) [#27861](https://github.com/nodejs/node/pull/27861) +* [[`e8fa0671a4`](https://github.com/nodejs/node/commit/e8fa0671a4)] - **tls**: destroy trace BIO instead of leaking it (Sam Roberts) [#27834](https://github.com/nodejs/node/pull/27834) +* [[`10e0d7f2ac`](https://github.com/nodejs/node/commit/10e0d7f2ac)] - **tls**: support the hints option (Luigi Pinca) [#27816](https://github.com/nodejs/node/pull/27816) +* [[`4716caa12e`](https://github.com/nodejs/node/commit/4716caa12e)] - **tls**: set tlsSocket.servername as early as possible (oyyd) [#27759](https://github.com/nodejs/node/pull/27759) +* [[`2ce24a9452`](https://github.com/nodejs/node/commit/2ce24a9452)] - **tools**: fix js2c regression (Refael Ackermann) [#27980](https://github.com/nodejs/node/pull/27980) +* [[`a75a59d3e3`](https://github.com/nodejs/node/commit/a75a59d3e3)] - **tools**: update inspector\_protocol to 0aafd2 (Michaël Zasso) [#27770](https://github.com/nodejs/node/pull/27770) +* [[`728bc2f59a`](https://github.com/nodejs/node/commit/728bc2f59a)] - **tools**: update dependencies in tools/doc (Rich Trott) [#27927](https://github.com/nodejs/node/pull/27927) +* [[`b54f3e0405`](https://github.com/nodejs/node/commit/b54f3e0405)] - **tools**: edit .eslintrc.js for minor maintainability improvements (Rich Trott) [#27789](https://github.com/nodejs/node/pull/27789) + ## 2019-05-22, Version 12.3.1 (Current), @BridgeAR diff --git a/doc/guides/writing-tests.md b/doc/guides/writing-tests.md index 7b1d168d85b2ff..b4d05e2811b254 100644 --- a/doc/guides/writing-tests.md +++ b/doc/guides/writing-tests.md @@ -424,6 +424,9 @@ will depend on what is being tested if this is required or not. To generate a test coverage report, see the [Test Coverage section of the Building guide][]. +Nightly coverage reports for the Node.js master branch are available at +https://coverage.nodejs.org/. + [ASCII]: http://man7.org/linux/man-pages/man7/ascii.7.html [Google Test]: https://github.com/google/googletest [`common` module]: https://github.com/nodejs/node/blob/master/test/common/README.md diff --git a/doc/node.1 b/doc/node.1 index ab715c0c1bfca2..14266a3f57382b 100644 --- a/doc/node.1 +++ b/doc/node.1 @@ -108,9 +108,6 @@ Requires Node.js to be built with .It Fl -es-module-specifier-resolution Select extension resolution algorithm for ES Modules; either 'explicit' (default) or 'node' . -.It Fl -experimental-json-modules -Enable experimental JSON interop support for the ES Module loader. -. .It Fl -experimental-modules Enable experimental ES module support and caching modules. . @@ -130,6 +127,9 @@ feature. .It Fl -experimental-vm-modules Enable experimental ES module support in VM module. . +.It Fl -experimental-wasm-modules +Enable experimental WebAssembly module support. +. .It Fl -force-fips Force FIPS-compliant crypto on startup (Cannot be disabled from script code). @@ -142,12 +142,37 @@ Enable experimental frozen intrinsics support. .It Fl -heapsnapshot-signal Ns = Ns Ar signal Generate heap snapshot on specified signal. . +.It Fl -heap-prof +Start the V8 heap profiler on start up, and write the heap profile to disk +before exit. If +.Fl -heap-prof-dir +is not specified, the profile will be written to the current working directory +with a generated file name. +. +.It Fl -heap-prof-dir +The directory where the heap profiles generated by +.Fl -heap-prof +will be placed. +. +.It Fl -heap-prof-interval +The average sampling interval in bytes for the heap profiles generated by +.Fl -heap-prof . +The default is +.Sy 512 * 1024 . +. +.It Fl -heap-prof-name +File name of the V8 heap profile generated with +.Fl -heap-prof +. .It Fl -http-parser Ns = Ns Ar library Chooses an HTTP parser library. Available values are .Sy llhttp or .Sy legacy . . +.It Fl -http-server-default-timeout Ns = Ns Ar milliseconds +Overrides the default value for server socket timeout. +. .It Fl -icu-data-dir Ns = Ns Ar file Specify ICU data load path. Overrides diff --git a/lib/_http_client.js b/lib/_http_client.js index bc2c2af8acd609..5555db13623553 100644 --- a/lib/_http_client.js +++ b/lib/_http_client.js @@ -40,13 +40,14 @@ const { Buffer } = require('buffer'); const { defaultTriggerAsyncIdScope } = require('internal/async_hooks'); const { URL, urlToOptions, searchParamsSymbol } = require('internal/url'); const { outHeadersKey, ondrain } = require('internal/http'); +const { connResetException, codes } = require('internal/errors'); const { ERR_HTTP_HEADERS_SENT, ERR_INVALID_ARG_TYPE, ERR_INVALID_HTTP_TOKEN, ERR_INVALID_PROTOCOL, ERR_UNESCAPED_CHARACTERS -} = require('internal/errors').codes; +} = codes; const { getTimerDuration } = require('internal/timers'); const { DTRACE_HTTP_CLIENT_REQUEST, @@ -337,15 +338,6 @@ function emitAbortNT() { this.emit('abort'); } - -function createHangUpError() { - // eslint-disable-next-line no-restricted-syntax - const error = new Error('socket hang up'); - error.code = 'ECONNRESET'; - return error; -} - - function socketCloseListener() { const socket = this; const req = socket._httpMessage; @@ -381,7 +373,7 @@ function socketCloseListener() { // receive a response. The error needs to // fire on the request. req.socket._hadError = true; - req.emit('error', createHangUpError()); + req.emit('error', connResetException('socket hang up')); } req.emit('close'); } @@ -441,7 +433,7 @@ function socketOnEnd() { // If we don't have a response then we know that the socket // ended prematurely and we need to emit an error on the request. req.socket._hadError = true; - req.emit('error', createHangUpError()); + req.emit('error', connResetException('socket hang up')); } if (parser) { parser.finish(); @@ -719,10 +711,10 @@ function onSocketNT(req, socket) { ClientRequest.prototype._deferToConnect = _deferToConnect; function _deferToConnect(method, arguments_, cb) { // This function is for calls that need to happen once the socket is - // connected and writable. It's an important promisy thing for all the socket - // calls that happen either now (when a socket is assigned) or - // in the future (when a socket gets assigned out of the pool and is - // eventually writable). + // assigned to this request and writable. It's an important promisy + // thing for all the socket calls that happen either now + // (when a socket is assigned) or in the future (when a socket gets + // assigned out of the pool and is eventually writable). const callSocketMethod = () => { if (method) diff --git a/lib/_http_outgoing.js b/lib/_http_outgoing.js index cb09e764fefdfb..a4a2b3ab144400 100644 --- a/lib/_http_outgoing.js +++ b/lib/_http_outgoing.js @@ -573,6 +573,7 @@ function write_(msg, chunk, encoding, callback, fromEnd) { if (!msg._hasBody) { debug('This type of response MUST NOT have a body. ' + 'Ignoring write() calls.'); + if (callback) process.nextTick(callback); return true; } diff --git a/lib/_http_server.js b/lib/_http_server.js index 4a76f1a0884c6f..ff69c70c9ca353 100644 --- a/lib/_http_server.js +++ b/lib/_http_server.js @@ -55,8 +55,11 @@ const { DTRACE_HTTP_SERVER_REQUEST, DTRACE_HTTP_SERVER_RESPONSE } = require('internal/dtrace'); +const { getOptionValue } = require('internal/options'); const kServerResponse = Symbol('ServerResponse'); +const kDefaultHttpServerTimeout = + getOptionValue('--http-server-default-timeout'); const STATUS_CODES = { 100: 'Continue', @@ -315,7 +318,7 @@ function Server(options, requestListener) { this.on('connection', connectionListener); - this.timeout = 2 * 60 * 1000; + this.timeout = kDefaultHttpServerTimeout; this.keepAliveTimeout = 5000; this.maxHeadersCount = null; this.headersTimeout = 40 * 1000; // 40 seconds @@ -404,9 +407,10 @@ function connectionListenerInternal(server, socket) { socket.on('resume', onSocketResume); socket.on('pause', onSocketPause); - // Override on to unconsume on `data`, `readable` listeners - socket.on = socketOnWrap; - socket.addListener = socket.on; + // Overrides to unconsume on `data`, `readable` listeners + socket.on = generateSocketListenerWrapper('on'); + socket.addListener = generateSocketListenerWrapper('addListener'); + socket.prependListener = generateSocketListenerWrapper('prependListener'); // We only consume the socket if it has never been consumed before. if (socket._handle && socket._handle.isStreamBase && @@ -754,19 +758,21 @@ function unconsume(parser, socket) { } } -function socketOnWrap(ev, fn) { - const res = net.Socket.prototype.on.call(this, ev, fn); - if (!this.parser) { - this.prependListener = net.Socket.prototype.prependListener; - this.on = net.Socket.prototype.on; - this.addListener = this.on; - return res; - } +function generateSocketListenerWrapper(originalFnName) { + return function socketListenerWrap(ev, fn) { + const res = net.Socket.prototype[originalFnName].call(this, ev, fn); + if (!this.parser) { + this.on = net.Socket.prototype.on; + this.addListener = net.Socket.prototype.addListener; + this.prependListener = net.Socket.prototype.prependListener; + return res; + } - if (ev === 'data' || ev === 'readable') - unconsume(this.parser, this); + if (ev === 'data' || ev === 'readable') + unconsume(this.parser, this); - return res; + return res; + }; } function resetHeadersTimeoutOnReqEnd() { diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index 5905c56bd4abd7..d6db7188750ebd 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -207,13 +207,28 @@ Readable.prototype._destroy = function(err, cb) { // similar to how Writable.write() returns true if you should // write() some more. Readable.prototype.push = function(chunk, encoding) { - const state = this._readableState; - var skipChunkCheck; + return readableAddChunk(this, chunk, encoding, false); +}; + +// Unshift should *always* be something directly out of read() +Readable.prototype.unshift = function(chunk, encoding) { + return readableAddChunk(this, chunk, encoding, true); +}; + +function readableAddChunk(stream, chunk, encoding, addToFront) { + debug('readableAddChunk', chunk); + const state = stream._readableState; + + let skipChunkCheck; if (!state.objectMode) { if (typeof chunk === 'string') { encoding = encoding || state.defaultEncoding; - if (encoding !== state.encoding) { + if (addToFront && state.encoding && state.encoding !== encoding) { + // When unshifting, if state.encoding is set, we have to save + // the string in the BufferList with the state encoding + chunk = Buffer.from(chunk, encoding).toString(state.encoding); + } else if (encoding !== state.encoding) { chunk = Buffer.from(chunk, encoding); encoding = ''; } @@ -223,17 +238,6 @@ Readable.prototype.push = function(chunk, encoding) { skipChunkCheck = true; } - return readableAddChunk(this, chunk, encoding, false, skipChunkCheck); -}; - -// Unshift should *always* be something directly out of read() -Readable.prototype.unshift = function(chunk) { - return readableAddChunk(this, chunk, null, true, false); -}; - -function readableAddChunk(stream, chunk, encoding, addToFront, skipChunkCheck) { - debug('readableAddChunk', chunk); - const state = stream._readableState; if (chunk === null) { state.reading = false; onEofChunk(stream, state); @@ -321,9 +325,22 @@ Readable.prototype.isPaused = function() { Readable.prototype.setEncoding = function(enc) { if (!StringDecoder) StringDecoder = require('string_decoder').StringDecoder; - this._readableState.decoder = new StringDecoder(enc); + const decoder = new StringDecoder(enc); + this._readableState.decoder = decoder; // If setEncoding(null), decoder.encoding equals utf8 this._readableState.encoding = this._readableState.decoder.encoding; + + // Iterate over current buffer to convert already stored Buffers: + let p = this._readableState.buffer.head; + let content = ''; + while (p !== null) { + content += decoder.write(p.data); + p = p.next; + } + this._readableState.buffer.clear(); + if (content !== '') + this._readableState.buffer.push(content); + this._readableState.length = content.length; return this; }; diff --git a/lib/_tls_wrap.js b/lib/_tls_wrap.js index 618f3892cee83b..58ad741cbf2172 100644 --- a/lib/_tls_wrap.js +++ b/lib/_tls_wrap.js @@ -44,6 +44,7 @@ const tls_wrap = internalBinding('tls_wrap'); const { Pipe, constants: PipeConstants } = internalBinding('pipe_wrap'); const { owner_symbol } = require('internal/async_hooks').symbols; const { SecureContext: NativeSecureContext } = internalBinding('crypto'); +const { connResetException, codes } = require('internal/errors'); const { ERR_INVALID_ARG_TYPE, ERR_INVALID_CALLBACK, @@ -55,7 +56,7 @@ const { ERR_TLS_REQUIRED_SERVER_NAME, ERR_TLS_SESSION_ATTACK, ERR_TLS_SNI_FROM_SERVER -} = require('internal/errors').codes; +} = codes; const { getOptionValue } = require('internal/options'); const { validateString } = require('internal/validators'); const traceTls = getOptionValue('--trace-tls'); @@ -442,7 +443,7 @@ const proxiedMethods = [ 'setSimultaneousAccepts', 'setBlocking', // PipeWrap - 'setPendingInstances' + 'setPendingInstances', ]; // Proxy HandleWrap, PipeWrap and TCPWrap methods @@ -774,7 +775,10 @@ TLSSocket.prototype._finishInit = function() { return; this.alpnProtocol = this._handle.getALPNNegotiatedProtocol(); - this.servername = this._handle.getServername(); + // The servername could be set by TLSWrap::SelectSNIContextCallback(). + if (this.servername === null) { + this.servername = this._handle.getServername(); + } debug('%s _finishInit', this._tlsOptions.isServer ? 'server' : 'client', @@ -905,9 +909,7 @@ function onSocketClose(err) { // Emit ECONNRESET if (!this._controlReleased && !this[kErrorEmitted]) { this[kErrorEmitted] = true; - // eslint-disable-next-line no-restricted-syntax - const connReset = new Error('socket hang up'); - connReset.code = 'ECONNRESET'; + const connReset = connResetException('socket hang up'); this._tlsOptions.server.emit('tlsClientError', connReset, this); } } @@ -1350,10 +1352,9 @@ function onConnectEnd() { if (!this._hadError) { const options = this[kConnectOptions]; this._hadError = true; - // eslint-disable-next-line no-restricted-syntax - const error = new Error('Client network socket disconnected before ' + - 'secure TLS connection was established'); - error.code = 'ECONNRESET'; + const error = connResetException('Client network socket disconnected ' + + 'before secure TLS connection was ' + + 'established'); error.path = options.path; error.host = options.host; error.port = options.port; @@ -1417,23 +1418,13 @@ exports.connect = function connect(...args) { tlssock.once('secureConnect', cb); if (!options.socket) { - // If user provided the socket, its their responsibility to manage its + // If user provided the socket, it's their responsibility to manage its // connectivity. If we created one internally, we connect it. - const connectOpt = { - path: options.path, - port: options.port, - host: options.host, - family: options.family, - localAddress: options.localAddress, - localPort: options.localPort, - lookup: options.lookup - }; - if (options.timeout) { tlssock.setTimeout(options.timeout); } - tlssock.connect(connectOpt, tlssock._start); + tlssock.connect(options, tlssock._start); } tlssock._releaseControl(); diff --git a/lib/child_process.js b/lib/child_process.js index 26965aa6b9c504..66be7611dc9587 100644 --- a/lib/child_process.js +++ b/lib/child_process.js @@ -51,9 +51,7 @@ const { const MAX_BUFFER = 1024 * 1024; -exports.ChildProcess = ChildProcess; - -exports.fork = function fork(modulePath /* , args, options */) { +function fork(modulePath /* , args, options */) { validateString(modulePath, 'modulePath'); // Get options and args arguments. @@ -108,10 +106,9 @@ exports.fork = function fork(modulePath /* , args, options */) { options.shell = false; return spawn(options.execPath, args, options); -}; - +} -exports._forkChild = function _forkChild(fd) { +function _forkChild(fd) { // set process.send() const p = new Pipe(PipeConstants.IPC); p.open(fd); @@ -123,8 +120,7 @@ exports._forkChild = function _forkChild(fd) { process.on('removeListener', function onRemoveListener(name) { if (name === 'message' || name === 'disconnect') control.unref(); }); -}; - +} function normalizeExecArgs(command, options, callback) { if (typeof options === 'function') { @@ -144,12 +140,12 @@ function normalizeExecArgs(command, options, callback) { } -exports.exec = function exec(command, options, callback) { +function exec(command, options, callback) { const opts = normalizeExecArgs(command, options, callback); - return exports.execFile(opts.file, - opts.options, - opts.callback); -}; + return module.exports.execFile(opts.file, + opts.options, + opts.callback); +} const customPromiseExecFunction = (orig) => { return (...args) => { @@ -167,12 +163,12 @@ const customPromiseExecFunction = (orig) => { }; }; -Object.defineProperty(exports.exec, promisify.custom, { +Object.defineProperty(exec, promisify.custom, { enumerable: false, - value: customPromiseExecFunction(exports.exec) + value: customPromiseExecFunction(exec) }); -exports.execFile = function execFile(file /* , args, options, callback */) { +function execFile(file /* , args, options, callback */) { let args = []; let callback; let options; @@ -386,11 +382,11 @@ exports.execFile = function execFile(file /* , args, options, callback */) { child.addListener('error', errorhandler); return child; -}; +} -Object.defineProperty(exports.execFile, promisify.custom, { +Object.defineProperty(execFile, promisify.custom, { enumerable: false, - value: customPromiseExecFunction(exports.execFile) + value: customPromiseExecFunction(execFile) }); function normalizeSpawnArguments(file, args, options) { @@ -529,7 +525,7 @@ function normalizeSpawnArguments(file, args, options) { } -var spawn = exports.spawn = function spawn(file, args, options) { +function spawn(file, args, options) { const opts = normalizeSpawnArguments(file, args, options); const child = new ChildProcess(); @@ -550,7 +546,7 @@ var spawn = exports.spawn = function spawn(file, args, options) { }); return child; -}; +} function spawnSync(file, args, options) { const opts = normalizeSpawnArguments(file, args, options); @@ -559,7 +555,7 @@ function spawnSync(file, args, options) { maxBuffer: MAX_BUFFER, ...opts.options }; - options = opts.options = Object.assign(defaults, opts.options); + options = opts.options = defaults; debug('spawnSync', opts.args, options); @@ -605,7 +601,6 @@ function spawnSync(file, args, options) { return child_process.spawnSync(opts); } -exports.spawnSync = spawnSync; function checkExecSyncError(ret, args, cmd) { @@ -643,7 +638,6 @@ function execFileSync(command, args, options) { return ret.stdout; } -exports.execFileSync = execFileSync; function execSync(command, options) { @@ -662,7 +656,6 @@ function execSync(command, options) { return ret.stdout; } -exports.execSync = execSync; function validateTimeout(timeout) { @@ -690,3 +683,15 @@ function sanitizeKillSignal(killSignal) { killSignal); } } + +module.exports = { + _forkChild, + ChildProcess, + exec, + execFile, + execFileSync, + execSync, + fork, + spawn, + spawnSync +}; diff --git a/lib/fs.js b/lib/fs.js index 7a31e26ccb1401..e890d0c1305b95 100644 --- a/lib/fs.js +++ b/lib/fs.js @@ -75,8 +75,7 @@ const { validateOffsetLengthRead, validateOffsetLengthWrite, validatePath, - warnOnNonPortableTemplate, - handleErrorFromBinding + warnOnNonPortableTemplate } = require('internal/fs/utils'); const { CHAR_FORWARD_SLASH, @@ -119,6 +118,23 @@ function showTruncateDeprecation() { } } +function handleErrorFromBinding(ctx) { + if (ctx.errno !== undefined) { // libuv error numbers + const err = uvException(ctx); + // eslint-disable-next-line no-restricted-syntax + Error.captureStackTrace(err, handleErrorFromBinding); + throw err; + } + if (ctx.error !== undefined) { // Errors created in C++ land. + // TODO(joyeecheung): currently, ctx.error are encoding errors + // usually caused by memory problems. We need to figure out proper error + // code(s) for this. + // eslint-disable-next-line no-restricted-syntax + Error.captureStackTrace(ctx.error, handleErrorFromBinding); + throw ctx.error; + } +} + function maybeCallback(cb) { if (typeof cb === 'function') return cb; @@ -1517,11 +1533,9 @@ realpathSync.native = (path, options) => { function realpath(p, options, callback) { callback = typeof options === 'function' ? options : maybeCallback(callback); - if (!options) - options = emptyObj; - else - options = getOptions(options, emptyObj); + options = getOptions(options, {}); p = toPathIfFileURL(p); + if (typeof p !== 'string') { p += ''; } diff --git a/lib/https.js b/lib/https.js index 4e649017312a24..44b885b0a42891 100644 --- a/lib/https.js +++ b/lib/https.js @@ -38,6 +38,10 @@ const debug = require('internal/util/debuglog').debuglog('https'); const { URL, urlToOptions, searchParamsSymbol } = require('internal/url'); const { IncomingMessage, ServerResponse } = require('http'); const { kIncomingMessage } = require('_http_common'); +const { getOptionValue } = require('internal/options'); + +const kDefaultHttpServerTimeout = + getOptionValue('--http-server-default-timeout'); function Server(opts, requestListener) { if (!(this instanceof Server)) return new Server(opts, requestListener); @@ -71,7 +75,7 @@ function Server(opts, requestListener) { conn.destroy(err); }); - this.timeout = 2 * 60 * 1000; + this.timeout = kDefaultHttpServerTimeout; this.keepAliveTimeout = 5000; this.maxHeadersCount = null; this.headersTimeout = 40 * 1000; // 40 seconds diff --git a/lib/inspector.js b/lib/inspector.js index 33c48125bbd22e..4bec628b7d789e 100644 --- a/lib/inspector.js +++ b/lib/inspector.js @@ -37,12 +37,8 @@ class Session extends EventEmitter { connect() { if (this[connectionSymbol]) throw new ERR_INSPECTOR_ALREADY_CONNECTED('The inspector session'); - const connection = + this[connectionSymbol] = new Connection((message) => this[onMessageSymbol](message)); - if (connection.sessionAttached) { - throw new ERR_INSPECTOR_ALREADY_CONNECTED('Another inspector session'); - } - this[connectionSymbol] = connection; } [onMessageSymbol](message) { diff --git a/lib/internal/assert/assertion_error.js b/lib/internal/assert/assertion_error.js index 7b62fca0352f5d..18d9951af0ddf0 100644 --- a/lib/internal/assert/assertion_error.js +++ b/lib/internal/assert/assertion_error.js @@ -374,9 +374,9 @@ class AssertionError extends Error { } else { let res = inspectValue(actual); let other = inspectValue(expected); - const knownOperators = kReadableOperator[operator]; + const knownOperator = kReadableOperator[operator]; if (operator === 'notDeepEqual' && res === other) { - res = `${knownOperators}\n\n${res}`; + res = `${knownOperator}\n\n${res}`; if (res.length > 1024) { res = `${res.slice(0, 1021)}...`; } @@ -389,13 +389,11 @@ class AssertionError extends Error { other = `${other.slice(0, 509)}...`; } if (operator === 'deepEqual') { - const eq = operator === 'deepEqual' ? 'deep-equal' : 'equal'; - res = `${knownOperators}\n\n${res}\n\nshould loosely ${eq}\n\n`; + res = `${knownOperator}\n\n${res}\n\nshould loosely deep-equal\n\n`; } else { - const newOperator = kReadableOperator[`${operator}Unequal`]; - if (newOperator) { - const eq = operator === 'notDeepEqual' ? 'deep-equal' : 'equal'; - res = `${newOperator}\n\n${res}\n\nshould not loosely ${eq}\n\n`; + const newOp = kReadableOperator[`${operator}Unequal`]; + if (newOp) { + res = `${newOp}\n\n${res}\n\nshould not loosely deep-equal\n\n`; } else { other = ` ${operator} ${other}`; } diff --git a/lib/internal/console/constructor.js b/lib/internal/console/constructor.js index 9c262aef62be25..e3de39f8126b7c 100644 --- a/lib/internal/console/constructor.js +++ b/lib/internal/console/constructor.js @@ -412,6 +412,7 @@ const consoleMethods = { const opt = { depth, maxArrayLength: 3, + breakLength: Infinity, ...this[kGetInspectOptions](this._stdout) }; return inspect(v, opt); diff --git a/lib/internal/crypto/keys.js b/lib/internal/crypto/keys.js index 321506eaf4a531..fb17ba36ced0e3 100644 --- a/lib/internal/crypto/keys.js +++ b/lib/internal/crypto/keys.js @@ -44,7 +44,7 @@ class KeyObject { if (type !== 'secret' && type !== 'public' && type !== 'private') throw new ERR_INVALID_ARG_VALUE('type', type); if (typeof handle !== 'object') - throw new ERR_INVALID_ARG_TYPE('handle', 'string', handle); + throw new ERR_INVALID_ARG_TYPE('handle', 'object', handle); this[kKeyType] = type; diff --git a/lib/internal/errors.js b/lib/internal/errors.js index 266358310bf3f7..f4f4ee09be3f97 100644 --- a/lib/internal/errors.js +++ b/lib/internal/errors.js @@ -554,6 +554,13 @@ function dnsException(code, syscall, hostname) { return ex; } +function connResetException(msg) { + // eslint-disable-next-line no-restricted-syntax + const ex = new Error(msg); + ex.code = 'ECONNRESET'; + return ex; +} + let maxStack_ErrorName; let maxStack_ErrorMessage; /** @@ -619,6 +626,7 @@ module.exports = { getMessage, hideStackFrames, isStackOverflowError, + connResetException, uvException, uvExceptionWithHostPort, SystemError, diff --git a/lib/internal/fs/utils.js b/lib/internal/fs/utils.js index 4cb06690bf558f..14abad81ec4e54 100644 --- a/lib/internal/fs/utils.js +++ b/lib/internal/fs/utils.js @@ -12,8 +12,7 @@ const { ERR_INVALID_OPT_VALUE_ENCODING, ERR_OUT_OF_RANGE }, - hideStackFrames, - uvException + hideStackFrames } = require('internal/errors'); const { isUint8Array, @@ -452,26 +451,7 @@ function warnOnNonPortableTemplate(template) { } } -// This handles errors following the convention of the fs binding. -function handleErrorFromBinding(ctx) { - if (ctx.errno !== undefined) { // libuv error numbers - const err = uvException(ctx); - // eslint-disable-next-line no-restricted-syntax - Error.captureStackTrace(err, handleErrorFromBinding); - throw err; - } - if (ctx.error !== undefined) { // Errors created in C++ land. - // TODO(joyeecheung): currently, ctx.error are encoding errors - // usually caused by memory problems. We need to figure out proper error - // code(s) for this. - // eslint-disable-next-line no-restricted-syntax - Error.captureStackTrace(ctx.error, handleErrorFromBinding); - throw ctx.error; - } -} - module.exports = { - handleErrorFromBinding, assertEncoding, copyObject, Dirent, diff --git a/lib/internal/http2/core.js b/lib/internal/http2/core.js index 3beab54ae4ca77..b1e5d4cef55392 100644 --- a/lib/internal/http2/core.js +++ b/lib/internal/http2/core.js @@ -137,6 +137,7 @@ const { UV_EOF } = internalBinding('uv'); const { StreamPipe } = internalBinding('stream_pipe'); const { _connectionListener: httpConnectionListener } = http; const debug = require('internal/util/debuglog').debuglog('http2'); +const { getOptionValue } = require('internal/options'); const kMaxFrameSize = (2 ** 24) - 1; const kMaxInt = (2 ** 32) - 1; @@ -171,7 +172,8 @@ const kState = Symbol('state'); const kType = Symbol('type'); const kWriteGeneric = Symbol('write-generic'); -const kDefaultSocketTimeout = 2 * 60 * 1000; +const kDefaultHttpServerTimeout = + getOptionValue('--http-server-default-timeout'); const { paddingBuffer, @@ -1071,6 +1073,9 @@ class Http2Session extends EventEmitter { } [kInspect](depth, opts) { + if (typeof depth === 'number' && depth < 0) + return this; + const obj = { type: this[kType], closed: this.closed, @@ -1647,6 +1652,9 @@ class Http2Stream extends Duplex { } [kInspect](depth, opts) { + if (typeof depth === 'number' && depth < 0) + return this; + const obj = { id: this[kID] || '', closed: this.closed, @@ -2679,7 +2687,7 @@ class Http2SecureServer extends TLSServer { options = initializeTLSOptions(options); super(options, connectionListener); this[kOptions] = options; - this.timeout = kDefaultSocketTimeout; + this.timeout = kDefaultHttpServerTimeout; this.on('newListener', setupCompat); if (typeof requestListener === 'function') this.on('request', requestListener); @@ -2699,9 +2707,10 @@ class Http2SecureServer extends TLSServer { class Http2Server extends NETServer { constructor(options, requestListener) { - super(connectionListener); - this[kOptions] = initializeOptions(options); - this.timeout = kDefaultSocketTimeout; + options = initializeOptions(options); + super(options, connectionListener); + this[kOptions] = options; + this.timeout = kDefaultHttpServerTimeout; this.on('newListener', setupCompat); if (typeof requestListener === 'function') this.on('request', requestListener); diff --git a/lib/internal/main/repl.js b/lib/internal/main/repl.js index b38102a15482fd..93b932f0bdd15f 100644 --- a/lib/internal/main/repl.js +++ b/lib/internal/main/repl.js @@ -11,7 +11,7 @@ const { evalScript } = require('internal/process/execution'); -const { print, kStderr, kStdout } = require('internal/util/print'); +const console = require('internal/console/global'); const { getOptionValue } = require('internal/options'); @@ -21,12 +21,14 @@ markBootstrapComplete(); // --input-type flag not supported in REPL if (getOptionValue('--input-type')) { - print(kStderr, 'Cannot specify --input-type for REPL'); + // If we can't write to stderr, we'd like to make this a noop, + // so use console.error. + console.error('Cannot specify --input-type for REPL'); process.exit(1); } -print(kStdout, `Welcome to Node.js ${process.version}.\n` + - 'Type ".help" for more information.'); +console.log(`Welcome to Node.js ${process.version}.\n` + + 'Type ".help" for more information.'); const cliRepl = require('internal/repl'); cliRepl.createInternalRepl(process.env, (err, repl) => { diff --git a/lib/internal/modules/esm/create_dynamic_module.js b/lib/internal/modules/esm/create_dynamic_module.js index 45f964d5ad8020..1201820003984f 100644 --- a/lib/internal/modules/esm/create_dynamic_module.js +++ b/lib/internal/modules/esm/create_dynamic_module.js @@ -4,24 +4,27 @@ const { ArrayPrototype, JSON, Object } = primordials; const debug = require('internal/util/debuglog').debuglog('esm'); -const createDynamicModule = (imports, exports, url = '', evaluate) => { - debug('creating ESM facade for %s with exports: %j', url, exports); - const names = ArrayPrototype.map(exports, (name) => `${name}`); - - const source = ` -${ArrayPrototype.join(ArrayPrototype.map(imports, (impt, index) => - `import * as $import_${index} from ${JSON.stringify(impt)}; -import.meta.imports[${JSON.stringify(impt)}] = $import_${index};`), '\n') +function createImport(impt, index) { + const imptPath = JSON.stringify(impt); + return `import * as $import_${index} from ${imptPath}; +import.meta.imports[${imptPath}] = $import_${index};`; } -${ArrayPrototype.join(ArrayPrototype.map(names, (name) => - `let $${name}; + +function createExport(expt) { + const name = `${expt}`; + return `let $${name}; export { $${name} as ${name} }; import.meta.exports.${name} = { get: () => $${name}, set: (v) => $${name} = v, -};`), '\n') +};`; } +const createDynamicModule = (imports, exports, url = '', evaluate) => { + debug('creating ESM facade for %s with exports: %j', url, exports); + const source = ` +${ArrayPrototype.join(ArrayPrototype.map(imports, createImport), '\n')} +${ArrayPrototype.join(ArrayPrototype.map(exports, createExport), '\n')} import.meta.done(); `; const { ModuleWrap, callbackMap } = internalBinding('module_wrap'); diff --git a/lib/internal/modules/esm/default_resolve.js b/lib/internal/modules/esm/default_resolve.js index 88377689ce4fd7..46e7b2415a92e0 100644 --- a/lib/internal/modules/esm/default_resolve.js +++ b/lib/internal/modules/esm/default_resolve.js @@ -8,7 +8,6 @@ const { getOptionValue } = require('internal/options'); const preserveSymlinks = getOptionValue('--preserve-symlinks'); const preserveSymlinksMain = getOptionValue('--preserve-symlinks-main'); -const experimentalJsonModules = getOptionValue('--experimental-json-modules'); const typeFlag = getOptionValue('--input-type'); const experimentalWasmModules = getOptionValue('--experimental-wasm-modules'); const { resolve: moduleWrapResolve, @@ -29,6 +28,7 @@ const extensionFormatMap = { '__proto__': null, '.cjs': 'commonjs', '.js': 'module', + '.json': 'json', '.mjs': 'module' }; @@ -36,7 +36,7 @@ const legacyExtensionFormatMap = { '__proto__': null, '.cjs': 'commonjs', '.js': 'commonjs', - '.json': 'commonjs', + '.json': 'json', '.mjs': 'module', '.node': 'commonjs' }; @@ -44,9 +44,6 @@ const legacyExtensionFormatMap = { if (experimentalWasmModules) extensionFormatMap['.wasm'] = legacyExtensionFormatMap['.wasm'] = 'wasm'; -if (experimentalJsonModules) - extensionFormatMap['.json'] = legacyExtensionFormatMap['.json'] = 'json'; - function resolve(specifier, parentURL) { if (NativeModule.canBeRequiredByUsers(specifier)) { return { diff --git a/lib/internal/process/execution.js b/lib/internal/process/execution.js index 227c1c2149cef5..2b7fc41ccddf01 100644 --- a/lib/internal/process/execution.js +++ b/lib/internal/process/execution.js @@ -35,22 +35,24 @@ function tryGetCwd() { } } -function evalModule(source, printResult) { +function evalModule(source, print) { + const { log, error } = require('internal/console/global'); const { decorateErrorStack } = require('internal/util'); const asyncESM = require('internal/process/esm_loader'); - const { kStdout, kStderr, print } = require('internal/util/print'); asyncESM.loaderPromise.then(async (loader) => { const { result } = await loader.eval(source); - if (printResult) { print(kStdout, result); } + if (print) { + log(result); + } }) .catch((e) => { decorateErrorStack(e); - print(kStderr, e); + error(e); process.exit(1); }); } -function evalScript(name, body, breakFirstLine, printResult) { +function evalScript(name, body, breakFirstLine, print) { const CJSModule = require('internal/modules/cjs/loader'); const { kVmBreakFirstLineSymbol } = require('internal/util'); @@ -76,9 +78,9 @@ function evalScript(name, body, breakFirstLine, printResult) { [kVmBreakFirstLineSymbol]: ${!!breakFirstLine} });\n`; const result = module._compile(script, `${name}-wrapper`); - if (printResult) { - const { kStdout, print } = require('internal/util/print'); - print(kStdout, result); + if (print) { + const { log } = require('internal/console/global'); + log(result); } if (origModule !== undefined) diff --git a/lib/internal/util/print.js b/lib/internal/util/print.js deleted file mode 100644 index 4c9327502ebad2..00000000000000 --- a/lib/internal/util/print.js +++ /dev/null @@ -1,67 +0,0 @@ -'use strict'; - -// This implements a light-weight printer that writes to stdout/stderr -// directly to avoid the overhead in the console abstraction. - -const { formatWithOptions } = require('internal/util/inspect'); -const { writeString } = internalBinding('fs'); -const { handleErrorFromBinding } = require('internal/fs/utils'); -const { guessHandleType } = internalBinding('util'); -const { log } = require('internal/console/global'); - -const kStdout = 1; -const kStderr = 2; -const handleType = [undefined, undefined, undefined]; -function getFdType(fd) { - if (handleType[fd] === undefined) { - handleType[fd] = guessHandleType(fd); - } - return handleType[fd]; -} - -function formatAndWrite(fd, obj, ignoreErrors, colors = false) { - const str = `${formatWithOptions({ colors }, obj)}\n`; - const ctx = {}; - writeString(fd, str, null, undefined, undefined, ctx); - if (!ignoreErrors) { - handleErrorFromBinding(ctx); - } -} - -let colors; -function getColors() { - if (colors === undefined) { - colors = require('internal/tty').getColorDepth() > 2; - } - return colors; -} - -// TODO(joyeecheung): replace more internal process._rawDebug() -// and console.log() usage with this if possible. -function print(fd, obj, ignoreErrors = true) { - switch (getFdType(fd)) { - case 'TTY': - formatAndWrite(fd, obj, ignoreErrors, getColors()); - break; - case 'FILE': - formatAndWrite(fd, obj, ignoreErrors); - break; - case 'PIPE': - case 'TCP': - // Fallback to console.log to handle IPC. - if (process.channel && process.channel.fd === fd) { - log(obj); - } else { - formatAndWrite(fd, obj, ignoreErrors); - } - break; - default: - log(obj); - } -} - -module.exports = { - print, - kStderr, - kStdout -}; diff --git a/lib/v8.js b/lib/v8.js index 2bede41291a947..cbb8229c7dd914 100644 --- a/lib/v8.js +++ b/lib/v8.js @@ -109,7 +109,9 @@ const { kSpaceSizeIndex, kSpaceUsedSizeIndex, kSpaceAvailableSizeIndex, - kPhysicalSpaceSizeIndex + kPhysicalSpaceSizeIndex, + kNumberOfNativeContextsIndex, + kNumberOfDetachedContextsIndex } = internalBinding('v8'); const kNumberOfHeapSpaces = kHeapSpaces.length; @@ -139,7 +141,9 @@ function getHeapStatistics() { 'heap_size_limit': buffer[kHeapSizeLimitIndex], 'malloced_memory': buffer[kMallocedMemoryIndex], 'peak_malloced_memory': buffer[kPeakMallocedMemoryIndex], - 'does_zap_garbage': buffer[kDoesZapGarbageIndex] + 'does_zap_garbage': buffer[kDoesZapGarbageIndex], + 'number_of_native_contexts': buffer[kNumberOfNativeContextsIndex], + 'number_of_detached_contexts': buffer[kNumberOfDetachedContextsIndex] }; } diff --git a/node.gyp b/node.gyp index df70d26f35f4e7..198a7ec166aa07 100644 --- a/node.gyp +++ b/node.gyp @@ -184,7 +184,6 @@ 'lib/internal/url.js', 'lib/internal/util.js', 'lib/internal/util/comparisons.js', - 'lib/internal/util/print.js', 'lib/internal/util/debuglog.js', 'lib/internal/util/inspect.js', 'lib/internal/util/inspector.js', @@ -283,6 +282,14 @@ 'ImageHasSafeExceptionHandlers': 'false', }, }, + + 'conditions': [ + ['OS=="aix"', { + 'ldflags': [ + '-Wl,-bnoerrmsg', + ], + }], + ], }, 'targets': [ @@ -1083,6 +1090,7 @@ 'test/cctest/test_node_postmortem_metadata.cc', 'test/cctest/test_environment.cc', 'test/cctest/test_linked_binding.cc', + 'test/cctest/test_per_process.cc', 'test/cctest/test_platform.cc', 'test/cctest/test_report_util.cc', 'test/cctest/test_traced_value.cc', diff --git a/src/api/environment.cc b/src/api/environment.cc index 5dfac00647eddd..eeeef7442d834a 100644 --- a/src/api/environment.cc +++ b/src/api/environment.cc @@ -1,4 +1,3 @@ -#include "env.h" #include "node.h" #include "node_context_data.h" #include "node_errors.h" diff --git a/src/base_object.h b/src/base_object.h index cb83462ff51e54..f616108a1d9486 100644 --- a/src/base_object.h +++ b/src/base_object.h @@ -24,7 +24,7 @@ #if defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS -#include "memory_tracker-inl.h" +#include "memory_tracker.h" #include "v8.h" #include // std::remove_reference diff --git a/src/cares_wrap.cc b/src/cares_wrap.cc index 07882a4212c6b2..0be02596f96945 100644 --- a/src/cares_wrap.cc +++ b/src/cares_wrap.cc @@ -23,6 +23,7 @@ #include "ares.h" #include "async_wrap-inl.h" #include "env-inl.h" +#include "memory_tracker-inl.h" #include "node.h" #include "req_wrap-inl.h" #include "util-inl.h" diff --git a/src/debug_utils.cc b/src/debug_utils.cc index b86710fba6e85b..a9bfa86b6d2aea 100644 --- a/src/debug_utils.cc +++ b/src/debug_utils.cc @@ -1,4 +1,5 @@ #include "debug_utils.h" +#include "env-inl.h" #include "util-inl.h" #ifdef __POSIX__ diff --git a/src/debug_utils.h b/src/debug_utils.h index ef5a4c0c47590c..db01cacba6a1b6 100644 --- a/src/debug_utils.h +++ b/src/debug_utils.h @@ -4,7 +4,7 @@ #if defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS #include "async_wrap.h" -#include "env-inl.h" +#include "env.h" #include #include diff --git a/src/diagnosticfilename-inl.h b/src/diagnosticfilename-inl.h new file mode 100644 index 00000000000000..58a3a933acc605 --- /dev/null +++ b/src/diagnosticfilename-inl.h @@ -0,0 +1,33 @@ +#ifndef SRC_DIAGNOSTICFILENAME_INL_H_ +#define SRC_DIAGNOSTICFILENAME_INL_H_ + +#if defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS + +#include "node_internals.h" +#include "env-inl.h" + +namespace node { + +inline DiagnosticFilename::DiagnosticFilename( + Environment* env, + const char* prefix, + const char* ext) : + filename_(MakeFilename(env->thread_id(), prefix, ext)) { +} + +inline DiagnosticFilename::DiagnosticFilename( + uint64_t thread_id, + const char* prefix, + const char* ext) : + filename_(MakeFilename(thread_id, prefix, ext)) { +} + +inline const char* DiagnosticFilename::operator*() const { + return filename_.c_str(); +} + +} // namespace node + +#endif // defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS + +#endif // SRC_DIAGNOSTICFILENAME_INL_H_ diff --git a/src/env-inl.h b/src/env-inl.h index 4765d0db98525a..2239412ccac9cb 100644 --- a/src/env-inl.h +++ b/src/env-inl.h @@ -689,6 +689,41 @@ inline const std::string& Environment::cpu_prof_dir() const { return cpu_prof_dir_; } +inline void Environment::set_heap_profiler_connection( + std::unique_ptr connection) { + CHECK_NULL(heap_profiler_connection_); + std::swap(heap_profiler_connection_, connection); +} + +inline profiler::V8HeapProfilerConnection* +Environment::heap_profiler_connection() { + return heap_profiler_connection_.get(); +} + +inline void Environment::set_heap_prof_name(const std::string& name) { + heap_prof_name_ = name; +} + +inline const std::string& Environment::heap_prof_name() const { + return heap_prof_name_; +} + +inline void Environment::set_heap_prof_dir(const std::string& dir) { + heap_prof_dir_ = dir; +} + +inline const std::string& Environment::heap_prof_dir() const { + return heap_prof_dir_; +} + +inline void Environment::set_heap_prof_interval(uint64_t interval) { + heap_prof_interval_ = interval; +} + +inline uint64_t Environment::heap_prof_interval() const { + return heap_prof_interval_; +} + #endif // HAVE_INSPECTOR inline std::shared_ptr Environment::inspector_host_port() { diff --git a/src/env.cc b/src/env.cc index 5f408f18f146a4..bc43fac963bbc3 100644 --- a/src/env.cc +++ b/src/env.cc @@ -1,6 +1,7 @@ #include "env.h" #include "async_wrap.h" +#include "memory_tracker-inl.h" #include "node_buffer.h" #include "node_context_data.h" #include "node_errors.h" diff --git a/src/env.h b/src/env.h index 5544ac44db5f8a..0c6dbe3c8f581f 100644 --- a/src/env.h +++ b/src/env.h @@ -73,6 +73,7 @@ class AgentWriterHandle; namespace profiler { class V8CoverageConnection; class V8CpuProfilerConnection; +class V8HeapProfilerConnection; } // namespace profiler #endif // HAVE_INSPECTOR @@ -1151,6 +1152,20 @@ class Environment : public MemoryRetainer { inline void set_cpu_prof_dir(const std::string& dir); inline const std::string& cpu_prof_dir() const; + + void set_heap_profiler_connection( + std::unique_ptr connection); + profiler::V8HeapProfilerConnection* heap_profiler_connection(); + + inline void set_heap_prof_name(const std::string& name); + inline const std::string& heap_prof_name() const; + + inline void set_heap_prof_dir(const std::string& dir); + inline const std::string& heap_prof_dir() const; + + inline void set_heap_prof_interval(uint64_t interval); + inline uint64_t heap_prof_interval() const; + #endif // HAVE_INSPECTOR private: @@ -1190,6 +1205,10 @@ class Environment : public MemoryRetainer { std::string cpu_prof_dir_; std::string cpu_prof_name_; uint64_t cpu_prof_interval_; + std::unique_ptr heap_profiler_connection_; + std::string heap_prof_dir_; + std::string heap_prof_name_; + uint64_t heap_prof_interval_; #endif // HAVE_INSPECTOR std::shared_ptr options_; diff --git a/src/heap_utils.cc b/src/heap_utils.cc index ee0665cfe279d8..8391f1de3726c1 100644 --- a/src/heap_utils.cc +++ b/src/heap_utils.cc @@ -1,4 +1,6 @@ +#include "diagnosticfilename-inl.h" #include "env-inl.h" +#include "memory_tracker-inl.h" #include "stream_base-inl.h" #include "util-inl.h" diff --git a/src/inspector/node_inspector.gypi b/src/inspector/node_inspector.gypi index 2ee8cfd7dafe3f..1d1dbefd866645 100644 --- a/src/inspector/node_inspector.gypi +++ b/src/inspector/node_inspector.gypi @@ -15,9 +15,12 @@ 'node_protocol_files': [ '<(protocol_tool_path)/lib/Allocator_h.template', '<(protocol_tool_path)/lib/Array_h.template', - '<(protocol_tool_path)/lib/Collections_h.template', + '<(protocol_tool_path)/lib/base_string_adapter_cc.template', + '<(protocol_tool_path)/lib/base_string_adapter_h.template', '<(protocol_tool_path)/lib/DispatcherBase_cpp.template', '<(protocol_tool_path)/lib/DispatcherBase_h.template', + '<(protocol_tool_path)/lib/encoding_cpp.template', + '<(protocol_tool_path)/lib/encoding_h.template', '<(protocol_tool_path)/lib/ErrorSupport_cpp.template', '<(protocol_tool_path)/lib/ErrorSupport_h.template', '<(protocol_tool_path)/lib/Forward_h.template', diff --git a/src/inspector/node_protocol.pdl b/src/inspector/node_protocol.pdl index 36d528b937a038..608521b467d9e4 100644 --- a/src/inspector/node_protocol.pdl +++ b/src/inspector/node_protocol.pdl @@ -71,6 +71,11 @@ experimental domain NodeWorker # Detaches from all running workers and disables attaching to new workers as they are started. command disable + # Detached from the worker with given sessionId. + command detach + parameters + SessionID sessionId + # Issued when attached to a worker. event attachedToWorker parameters diff --git a/src/inspector/node_string.cc b/src/inspector/node_string.cc index a79df9e817c049..0d403c66f0197b 100644 --- a/src/inspector/node_string.cc +++ b/src/inspector/node_string.cc @@ -107,6 +107,22 @@ String fromUTF8(const uint8_t* data, size_t length) { return std::string(reinterpret_cast(data), length); } +String fromUTF16(const uint16_t* data, size_t length) { + icu::UnicodeString utf16(reinterpret_cast(data), length); + std::string result; + return utf16.toUTF8String(result); +} + +const uint8_t* CharactersUTF8(const String& s) { + return reinterpret_cast(s.data()); +} + +size_t CharacterCount(const String& s) { + icu::UnicodeString utf16 = + icu::UnicodeString::fromUTF8(icu::StringPiece(s.data(), s.length())); + return utf16.countChar32(); +} + } // namespace StringUtil } // namespace protocol } // namespace inspector diff --git a/src/inspector/node_string.h b/src/inspector/node_string.h index 39545b75aec334..1b8560b6fa5642 100644 --- a/src/inspector/node_string.h +++ b/src/inspector/node_string.h @@ -20,16 +20,6 @@ using String = std::string; using StringBuilder = std::ostringstream; using ProtocolMessage = std::string; -class StringUTF8Adapter { - public: - explicit StringUTF8Adapter(const std::string& string) : string_(string) { } - const char* Data() const { return string_.data(); } - size_t length() const { return string_.length(); } - - private: - const std::string& string_; -}; - namespace StringUtil { // NOLINTNEXTLINE(runtime/references) This is V8 API... inline void builderAppend(StringBuilder& builder, char c) { @@ -82,6 +72,13 @@ std::unique_ptr parseMessage(const std::string& message, bool binary); ProtocolMessage jsonToMessage(String message); ProtocolMessage binaryToMessage(std::vector message); String fromUTF8(const uint8_t* data, size_t length); +String fromUTF16(const uint16_t* data, size_t length); +const uint8_t* CharactersUTF8(const String& s); +size_t CharacterCount(const String& s); + +// Unimplemented. The generated code will fall back to CharactersUTF8(). +inline uint8_t* CharactersLatin1(const String& s) { return nullptr; } +inline const uint16_t* CharactersUTF16(const String& s) { return nullptr; } extern size_t kNotFound; } // namespace StringUtil diff --git a/src/inspector/runtime_agent.cc b/src/inspector/runtime_agent.cc index f8fdbe42d41e03..4056140e703493 100644 --- a/src/inspector/runtime_agent.cc +++ b/src/inspector/runtime_agent.cc @@ -16,10 +16,6 @@ void RuntimeAgent::Wire(UberDispatcher* dispatcher) { } DispatchResponse RuntimeAgent::notifyWhenWaitingForDisconnect(bool enabled) { - if (!env_->owns_process_state()) { - return DispatchResponse::Error( - "NodeRuntime domain can only be used through main thread sessions"); - } notify_when_waiting_for_disconnect_ = enabled; return DispatchResponse::OK(); } diff --git a/src/inspector/tracing_agent.cc b/src/inspector/tracing_agent.cc index d87eec6a6469b9..14f55d0cac0ff7 100644 --- a/src/inspector/tracing_agent.cc +++ b/src/inspector/tracing_agent.cc @@ -2,9 +2,6 @@ #include "main_thread_interface.h" #include "node_internals.h" #include "node_v8_platform-inl.h" - -#include "env-inl.h" -#include "util-inl.h" #include "v8.h" #include @@ -73,7 +70,7 @@ class SendMessageRequest : public Request { if (frontend_wrapper == nullptr) return; auto frontend = frontend_wrapper->get(); if (frontend != nullptr) { - frontend->sendRawNotification(message_); + frontend->sendRawJSONNotification(message_); } } diff --git a/src/inspector/worker_agent.cc b/src/inspector/worker_agent.cc index d343de8494a36f..81706572e646ea 100644 --- a/src/inspector/worker_agent.cc +++ b/src/inspector/worker_agent.cc @@ -115,6 +115,11 @@ DispatchResponse WorkerAgent::disable() { return DispatchResponse::OK(); } +DispatchResponse WorkerAgent::detach(const String& sessionId) { + workers_->Detached(sessionId); + return DispatchResponse::OK(); +} + void NodeWorkers::WorkerCreated(const std::string& title, const std::string& url, bool waiting, diff --git a/src/inspector/worker_agent.h b/src/inspector/worker_agent.h index 402c7194163967..1bd25189bf3026 100644 --- a/src/inspector/worker_agent.h +++ b/src/inspector/worker_agent.h @@ -25,6 +25,7 @@ class WorkerAgent : public NodeWorker::Backend { DispatchResponse enable(bool waitForDebuggerOnStart) override; DispatchResponse disable() override; + DispatchResponse detach(const String& sessionId) override; private: std::shared_ptr frontend_; diff --git a/src/inspector_agent.cc b/src/inspector_agent.cc index d8b5d01a285834..8a12c2dddf7493 100644 --- a/src/inspector_agent.cc +++ b/src/inspector_agent.cc @@ -1,5 +1,6 @@ #include "inspector_agent.h" +#include "env-inl.h" #include "inspector/main_thread_interface.h" #include "inspector/node_string.h" #include "inspector/runtime_agent.h" @@ -24,6 +25,7 @@ #include // PTHREAD_STACK_MIN #endif // __POSIX__ +#include #include #include #include @@ -110,12 +112,18 @@ static int StartDebugSignalHandler() { CHECK_EQ(0, uv_sem_init(&start_io_thread_semaphore, 0)); pthread_attr_t attr; CHECK_EQ(0, pthread_attr_init(&attr)); - // Don't shrink the thread's stack on FreeBSD. Said platform decided to - // follow the pthreads specification to the letter rather than in spirit: - // https://lists.freebsd.org/pipermail/freebsd-current/2014-March/048885.html -#ifndef __FreeBSD__ - CHECK_EQ(0, pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN)); -#endif // __FreeBSD__ +#if defined(PTHREAD_STACK_MIN) && !defined(__FreeBSD__) + // PTHREAD_STACK_MIN is 2 KB with musl libc, which is too small to safely + // receive signals. PTHREAD_STACK_MIN + MINSIGSTKSZ is 8 KB on arm64, which + // is the musl architecture with the biggest MINSIGSTKSZ so let's use that + // as a lower bound and let's quadruple it just in case. The goal is to avoid + // creating a big 2 or 4 MB address space gap (problematic on 32 bits + // because of fragmentation), not squeeze out every last byte. + // Omitted on FreeBSD because it doesn't seem to like small stacks. + const size_t stack_size = std::max(static_cast(4 * 8192), + static_cast(PTHREAD_STACK_MIN)); + CHECK_EQ(0, pthread_attr_setstacksize(&attr, stack_size)); +#endif // defined(PTHREAD_STACK_MIN) && !defined(__FreeBSD__) CHECK_EQ(0, pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED)); sigset_t sigmask; // Mask all signals. @@ -464,8 +472,8 @@ class NodeInspectorClient : public V8InspectorClient { runMessageLoop(); } - void waitForIoShutdown() { - waiting_for_io_shutdown_ = true; + void waitForSessionsDisconnect() { + waiting_for_sessions_disconnect_ = true; runMessageLoop(); } @@ -540,6 +548,8 @@ class NodeInspectorClient : public V8InspectorClient { } contextDestroyed(env_->context()); } + if (waiting_for_sessions_disconnect_ && !is_main_) + waiting_for_sessions_disconnect_ = false; } void dispatchMessageFromFrontend(int session_id, const StringView& message) { @@ -670,8 +680,9 @@ class NodeInspectorClient : public V8InspectorClient { bool shouldRunMessageLoop() { if (waiting_for_frontend_) return true; - if (waiting_for_io_shutdown_ || waiting_for_resume_) + if (waiting_for_sessions_disconnect_ || waiting_for_resume_) { return hasConnectedSessions(); + } return false; } @@ -715,7 +726,7 @@ class NodeInspectorClient : public V8InspectorClient { int next_session_id_ = 1; bool waiting_for_resume_ = false; bool waiting_for_frontend_ = false; - bool waiting_for_io_shutdown_ = false; + bool waiting_for_sessions_disconnect_ = false; // Allows accessing Inspector from non-main threads std::unique_ptr interface_; std::shared_ptr worker_manager_; @@ -811,11 +822,14 @@ void Agent::WaitForDisconnect() { fprintf(stderr, "Waiting for the debugger to disconnect...\n"); fflush(stderr); } - if (!client_->notifyWaitingForDisconnect()) + if (!client_->notifyWaitingForDisconnect()) { client_->contextDestroyed(parent_env_->context()); + } else if (is_worker) { + client_->waitForSessionsDisconnect(); + } if (io_ != nullptr) { io_->StopAcceptingNewConnections(); - client_->waitForIoShutdown(); + client_->waitForSessionsDisconnect(); } } diff --git a/src/inspector_io.cc b/src/inspector_io.cc index 7ba19087d01b0d..223e3592a1fc09 100644 --- a/src/inspector_io.cc +++ b/src/inspector_io.cc @@ -4,7 +4,6 @@ #include "inspector/main_thread_interface.h" #include "inspector/node_string.h" #include "base_object-inl.h" -#include "env-inl.h" #include "debug_utils.h" #include "node.h" #include "node_crypto.h" diff --git a/src/inspector_js_api.cc b/src/inspector_js_api.cc index 5caf3fa09a4d10..9d649385706131 100644 --- a/src/inspector_js_api.cc +++ b/src/inspector_js_api.cc @@ -1,6 +1,7 @@ #include "base_object-inl.h" #include "inspector_agent.h" #include "inspector_io.h" +#include "memory_tracker-inl.h" #include "util-inl.h" #include "v8.h" #include "v8-inspector.h" diff --git a/src/inspector_profiler.cc b/src/inspector_profiler.cc index 4604e0a5b441e1..1b20398dba4d3d 100644 --- a/src/inspector_profiler.cc +++ b/src/inspector_profiler.cc @@ -2,6 +2,8 @@ #include #include "base_object-inl.h" #include "debug_utils.h" +#include "diagnosticfilename-inl.h" +#include "memory_tracker-inl.h" #include "node_file.h" #include "node_internals.h" #include "v8-inspector.h" @@ -256,6 +258,44 @@ void V8CpuProfilerConnection::End() { DispatchMessage("Profiler.stop"); } +std::string V8HeapProfilerConnection::GetDirectory() const { + return env()->heap_prof_dir(); +} + +std::string V8HeapProfilerConnection::GetFilename() const { + return env()->heap_prof_name(); +} + +MaybeLocal V8HeapProfilerConnection::GetProfile(Local result) { + Local profile_v; + if (!result + ->Get(env()->context(), + FIXED_ONE_BYTE_STRING(env()->isolate(), "profile")) + .ToLocal(&profile_v)) { + fprintf(stderr, "'profile' from heap profile result is undefined\n"); + return MaybeLocal(); + } + if (!profile_v->IsObject()) { + fprintf(stderr, "'profile' from heap profile result is not an Object\n"); + return MaybeLocal(); + } + return profile_v.As(); +} + +void V8HeapProfilerConnection::Start() { + DispatchMessage("HeapProfiler.enable"); + std::string params = R"({ "samplingInterval": )"; + params += std::to_string(env()->heap_prof_interval()); + params += " }"; + DispatchMessage("HeapProfiler.startSampling", params.c_str()); +} + +void V8HeapProfilerConnection::End() { + CHECK_EQ(ending_, false); + ending_ = true; + DispatchMessage("HeapProfiler.stopSampling"); +} + // For now, we only support coverage profiling, but we may add more // in the future. void EndStartedProfilers(Environment* env) { @@ -266,6 +306,12 @@ void EndStartedProfilers(Environment* env) { connection->End(); } + connection = env->heap_profiler_connection(); + if (connection != nullptr && !connection->ending()) { + Debug(env, DebugCategory::INSPECTOR_PROFILER, "Ending heap profiling\n"); + connection->End(); + } + connection = env->coverage_connection(); if (connection != nullptr && !connection->ending()) { Debug( @@ -311,6 +357,20 @@ void StartProfilers(Environment* env) { std::make_unique(env)); env->cpu_profiler_connection()->Start(); } + if (env->options()->heap_prof) { + const std::string& dir = env->options()->heap_prof_dir; + env->set_heap_prof_interval(env->options()->heap_prof_interval); + env->set_heap_prof_dir(dir.empty() ? GetCwd() : dir); + if (env->options()->heap_prof_name.empty()) { + DiagnosticFilename filename(env, "Heap", "heapprofile"); + env->set_heap_prof_name(*filename); + } else { + env->set_heap_prof_name(env->options()->heap_prof_name); + } + env->set_heap_profiler_connection( + std::make_unique(env)); + env->heap_profiler_connection()->Start(); + } } static void SetCoverageDirectory(const FunctionCallbackInfo& args) { diff --git a/src/inspector_profiler.h b/src/inspector_profiler.h index 345ef90d4e15c6..e7d45d7de34f35 100644 --- a/src/inspector_profiler.h +++ b/src/inspector_profiler.h @@ -107,6 +107,26 @@ class V8CpuProfilerConnection : public V8ProfilerConnection { bool ending_ = false; }; +class V8HeapProfilerConnection : public V8ProfilerConnection { + public: + explicit V8HeapProfilerConnection(Environment* env) + : V8ProfilerConnection(env) {} + + void Start() override; + void End() override; + + const char* type() const override { return "heap"; } + bool ending() const override { return ending_; } + + std::string GetDirectory() const override; + std::string GetFilename() const override; + v8::MaybeLocal GetProfile(v8::Local result) override; + + private: + std::unique_ptr session_; + bool ending_ = false; +}; + } // namespace profiler } // namespace node diff --git a/src/js_native_api_v8.cc b/src/js_native_api_v8.cc index 286086ab6af46d..413231dd36c088 100644 --- a/src/js_native_api_v8.cc +++ b/src/js_native_api_v8.cc @@ -2,6 +2,7 @@ #include #include #define NAPI_EXPERIMENTAL +#include "env-inl.h" #include "js_native_api_v8.h" #include "js_native_api.h" #include "util-inl.h" @@ -2138,21 +2139,6 @@ napi_status napi_get_value_string_utf16(napi_env env, return napi_clear_last_error(env); } -napi_status napi_coerce_to_object(napi_env env, - napi_value value, - napi_value* result) { - NAPI_PREAMBLE(env); - CHECK_ARG(env, value); - CHECK_ARG(env, result); - - v8::Local context = env->context(); - v8::Local obj; - CHECK_TO_OBJECT(env, context, obj, value); - - *result = v8impl::JsValueFromV8LocalValue(obj); - return GET_RETURN_STATUS(env); -} - napi_status napi_coerce_to_bool(napi_env env, napi_value value, napi_value* result) { @@ -2167,37 +2153,28 @@ napi_status napi_coerce_to_bool(napi_env env, return GET_RETURN_STATUS(env); } -napi_status napi_coerce_to_number(napi_env env, - napi_value value, - napi_value* result) { - NAPI_PREAMBLE(env); - CHECK_ARG(env, value); - CHECK_ARG(env, result); - - v8::Local context = env->context(); - v8::Local num; - - CHECK_TO_NUMBER(env, context, num, value); - - *result = v8impl::JsValueFromV8LocalValue(num); - return GET_RETURN_STATUS(env); -} - -napi_status napi_coerce_to_string(napi_env env, - napi_value value, - napi_value* result) { - NAPI_PREAMBLE(env); - CHECK_ARG(env, value); - CHECK_ARG(env, result); - - v8::Local context = env->context(); - v8::Local str; - - CHECK_TO_STRING(env, context, str, value); - - *result = v8impl::JsValueFromV8LocalValue(str); - return GET_RETURN_STATUS(env); -} +#define GEN_COERCE_FUNCTION(UpperCaseName, MixedCaseName, LowerCaseName) \ + napi_status napi_coerce_to_##LowerCaseName(napi_env env, \ + napi_value value, \ + napi_value* result) { \ + NAPI_PREAMBLE(env); \ + CHECK_ARG(env, value); \ + CHECK_ARG(env, result); \ + \ + v8::Local context = env->context(); \ + v8::Local str; \ + \ + CHECK_TO_##UpperCaseName(env, context, str, value); \ + \ + *result = v8impl::JsValueFromV8LocalValue(str); \ + return GET_RETURN_STATUS(env); \ + } + +GEN_COERCE_FUNCTION(NUMBER, Number, number) +GEN_COERCE_FUNCTION(OBJECT, Object, object) +GEN_COERCE_FUNCTION(STRING, String, string) + +#undef GEN_COERCE_FUNCTION napi_status napi_wrap(napi_env env, napi_value js_object, diff --git a/src/js_stream.cc b/src/js_stream.cc index 1d61605d6459d4..2663106ba7af51 100644 --- a/src/js_stream.cc +++ b/src/js_stream.cc @@ -167,9 +167,9 @@ void JSStream::ReadBuffer(const FunctionCallbackInfo& args) { JSStream* wrap; ASSIGN_OR_RETURN_UNWRAP(&wrap, args.Holder()); - CHECK(Buffer::HasInstance(args[0])); - char* data = Buffer::Data(args[0]); - int len = Buffer::Length(args[0]); + ArrayBufferViewContents buffer(args[0]); + const char* data = buffer.data(); + int len = buffer.length(); // Repeatedly ask the stream's owner for memory, copy the data that we // just read from JS into those buffers and emit them as reads. diff --git a/src/module_wrap.cc b/src/module_wrap.cc index a4e81dcc29474b..e104afb736c28d 100644 --- a/src/module_wrap.cc +++ b/src/module_wrap.cc @@ -1,6 +1,7 @@ #include "module_wrap.h" #include "env.h" +#include "memory_tracker-inl.h" #include "node_errors.h" #include "node_url.h" #include "util-inl.h" diff --git a/src/node.cc b/src/node.cc index 953465e04ac9e4..f8b6fa6f33b673 100644 --- a/src/node.cc +++ b/src/node.cc @@ -25,6 +25,7 @@ #include "debug_utils.h" #include "env-inl.h" +#include "memory_tracker-inl.h" #include "node_binding.h" #include "node_internals.h" #include "node_main_instance.h" @@ -491,7 +492,7 @@ inline void PlatformInit() { for (unsigned nr = 1; nr < kMaxSignal; nr += 1) { if (nr == SIGKILL || nr == SIGSTOP) continue; - act.sa_handler = (nr == SIGPIPE) ? SIG_IGN : SIG_DFL; + act.sa_handler = (nr == SIGPIPE || nr == SIGXFSZ) ? SIG_IGN : SIG_DFL; CHECK_EQ(0, sigaction(nr, &act, nullptr)); } #endif // !NODE_SHARED_MODE diff --git a/src/node_api.cc b/src/node_api.cc index f8b0d8b550391f..91e6a14cdc1570 100644 --- a/src/node_api.cc +++ b/src/node_api.cc @@ -1,11 +1,12 @@ -#include -#include "env.h" +#include "env-inl.h" #define NAPI_EXPERIMENTAL #include "js_native_api_v8.h" #include "node_api.h" #include "node_binding.h" +#include "node_buffer.h" #include "node_errors.h" #include "node_internals.h" +#include "threadpoolwork-inl.h" #include "util-inl.h" #include diff --git a/src/node_buffer.cc b/src/node_buffer.cc index 3b4be5a8105f62..e6a88f649895e8 100644 --- a/src/node_buffer.cc +++ b/src/node_buffer.cc @@ -463,17 +463,17 @@ void StringSlice(const FunctionCallbackInfo& args) { Isolate* isolate = env->isolate(); THROW_AND_RETURN_UNLESS_BUFFER(env, args.This()); - SPREAD_BUFFER_ARG(args.This(), ts_obj); + ArrayBufferViewContents buffer(args.This()); - if (ts_obj_length == 0) + if (buffer.length() == 0) return args.GetReturnValue().SetEmptyString(); - SLICE_START_END(env, args[0], args[1], ts_obj_length) + SLICE_START_END(env, args[0], args[1], buffer.length()) Local error; MaybeLocal ret = StringBytes::Encode(isolate, - ts_obj_data + start, + buffer.data() + start, length, encoding, &error); @@ -492,9 +492,8 @@ void Copy(const FunctionCallbackInfo &args) { THROW_AND_RETURN_UNLESS_BUFFER(env, args[0]); THROW_AND_RETURN_UNLESS_BUFFER(env, args[1]); - Local buffer_obj = args[0].As(); + ArrayBufferViewContents source(args[0]); Local target_obj = args[1].As(); - SPREAD_BUFFER_ARG(buffer_obj, ts_obj); SPREAD_BUFFER_ARG(target_obj, target); size_t target_start = 0; @@ -503,14 +502,14 @@ void Copy(const FunctionCallbackInfo &args) { THROW_AND_RETURN_IF_OOB(ParseArrayIndex(env, args[2], 0, &target_start)); THROW_AND_RETURN_IF_OOB(ParseArrayIndex(env, args[3], 0, &source_start)); - THROW_AND_RETURN_IF_OOB(ParseArrayIndex(env, args[4], ts_obj_length, + THROW_AND_RETURN_IF_OOB(ParseArrayIndex(env, args[4], source.length(), &source_end)); // Copy 0 bytes; we're done if (target_start >= target_length || source_start >= source_end) return args.GetReturnValue().Set(0); - if (source_start > ts_obj_length) + if (source_start > source.length()) return THROW_ERR_OUT_OF_RANGE( env, "The value of \"sourceStart\" is out of range."); @@ -519,9 +518,9 @@ void Copy(const FunctionCallbackInfo &args) { uint32_t to_copy = std::min( std::min(source_end - source_start, target_length - target_start), - ts_obj_length - source_start); + source.length() - source_start); - memmove(target_data + target_start, ts_obj_data + source_start, to_copy); + memmove(target_data + target_start, source.data() + source_start, to_copy); args.GetReturnValue().Set(to_copy); } @@ -689,8 +688,8 @@ void CompareOffset(const FunctionCallbackInfo &args) { THROW_AND_RETURN_UNLESS_BUFFER(env, args[0]); THROW_AND_RETURN_UNLESS_BUFFER(env, args[1]); - SPREAD_BUFFER_ARG(args[0], ts_obj); - SPREAD_BUFFER_ARG(args[1], target); + ArrayBufferViewContents source(args[0]); + ArrayBufferViewContents target(args[1]); size_t target_start = 0; size_t source_start = 0; @@ -699,15 +698,15 @@ void CompareOffset(const FunctionCallbackInfo &args) { THROW_AND_RETURN_IF_OOB(ParseArrayIndex(env, args[2], 0, &target_start)); THROW_AND_RETURN_IF_OOB(ParseArrayIndex(env, args[3], 0, &source_start)); - THROW_AND_RETURN_IF_OOB(ParseArrayIndex(env, args[4], target_length, + THROW_AND_RETURN_IF_OOB(ParseArrayIndex(env, args[4], target.length(), &target_end)); - THROW_AND_RETURN_IF_OOB(ParseArrayIndex(env, args[5], ts_obj_length, + THROW_AND_RETURN_IF_OOB(ParseArrayIndex(env, args[5], source.length(), &source_end)); - if (source_start > ts_obj_length) + if (source_start > source.length()) return THROW_ERR_OUT_OF_RANGE( env, "The value of \"sourceStart\" is out of range."); - if (target_start > target_length) + if (target_start > target.length()) return THROW_ERR_OUT_OF_RANGE( env, "The value of \"targetStart\" is out of range."); @@ -716,11 +715,11 @@ void CompareOffset(const FunctionCallbackInfo &args) { size_t to_cmp = std::min(std::min(source_end - source_start, target_end - target_start), - ts_obj_length - source_start); + source.length() - source_start); int val = normalizeCompareVal(to_cmp > 0 ? - memcmp(ts_obj_data + source_start, - target_data + target_start, + memcmp(source.data() + source_start, + target.data() + target_start, to_cmp) : 0, source_end - source_start, target_end - target_start); @@ -733,14 +732,14 @@ void Compare(const FunctionCallbackInfo &args) { THROW_AND_RETURN_UNLESS_BUFFER(env, args[0]); THROW_AND_RETURN_UNLESS_BUFFER(env, args[1]); - SPREAD_BUFFER_ARG(args[0], obj_a); - SPREAD_BUFFER_ARG(args[1], obj_b); + ArrayBufferViewContents a(args[0]); + ArrayBufferViewContents b(args[1]); - size_t cmp_length = std::min(obj_a_length, obj_b_length); + size_t cmp_length = std::min(a.length(), b.length()); int val = normalizeCompareVal(cmp_length > 0 ? - memcmp(obj_a_data, obj_b_data, cmp_length) : 0, - obj_a_length, obj_b_length); + memcmp(a.data(), b.data(), cmp_length) : 0, + a.length(), b.length()); args.GetReturnValue().Set(val); } @@ -792,16 +791,16 @@ void IndexOfString(const FunctionCallbackInfo& args) { enum encoding enc = ParseEncoding(isolate, args[3], UTF8); THROW_AND_RETURN_UNLESS_BUFFER(env, args[0]); - SPREAD_BUFFER_ARG(args[0], ts_obj); + ArrayBufferViewContents buffer(args[0]); Local needle = args[1].As(); int64_t offset_i64 = args[2].As()->Value(); bool is_forward = args[4]->IsTrue(); - const char* haystack = ts_obj_data; + const char* haystack = buffer.data(); // Round down to the nearest multiple of 2 in case of UCS2. const size_t haystack_length = (enc == UCS2) ? - ts_obj_length &~ 1 : ts_obj_length; // NOLINT(whitespace/operators) + buffer.length() &~ 1 : buffer.length(); // NOLINT(whitespace/operators) size_t needle_length; if (!StringBytes::Size(isolate, needle, enc).To(&needle_length)) return; @@ -909,15 +908,15 @@ void IndexOfBuffer(const FunctionCallbackInfo& args) { THROW_AND_RETURN_UNLESS_BUFFER(Environment::GetCurrent(args), args[0]); THROW_AND_RETURN_UNLESS_BUFFER(Environment::GetCurrent(args), args[1]); - SPREAD_BUFFER_ARG(args[0], ts_obj); - SPREAD_BUFFER_ARG(args[1], buf); + ArrayBufferViewContents haystack_contents(args[0]); + ArrayBufferViewContents needle_contents(args[1]); int64_t offset_i64 = args[2].As()->Value(); bool is_forward = args[4]->IsTrue(); - const char* haystack = ts_obj_data; - const size_t haystack_length = ts_obj_length; - const char* needle = buf_data; - const size_t needle_length = buf_length; + const char* haystack = haystack_contents.data(); + const size_t haystack_length = haystack_contents.length(); + const char* needle = needle_contents.data(); + const size_t needle_length = needle_contents.length(); int64_t opt_offset = IndexOfOffset(haystack_length, offset_i64, @@ -978,27 +977,28 @@ void IndexOfNumber(const FunctionCallbackInfo& args) { CHECK(args[3]->IsBoolean()); THROW_AND_RETURN_UNLESS_BUFFER(Environment::GetCurrent(args), args[0]); - SPREAD_BUFFER_ARG(args[0], ts_obj); + ArrayBufferViewContents buffer(args[0]); uint32_t needle = args[1].As()->Value(); int64_t offset_i64 = args[2].As()->Value(); bool is_forward = args[3]->IsTrue(); - int64_t opt_offset = IndexOfOffset(ts_obj_length, offset_i64, 1, is_forward); - if (opt_offset <= -1 || ts_obj_length == 0) { + int64_t opt_offset = + IndexOfOffset(buffer.length(), offset_i64, 1, is_forward); + if (opt_offset <= -1 || buffer.length() == 0) { return args.GetReturnValue().Set(-1); } size_t offset = static_cast(opt_offset); - CHECK_LT(offset, ts_obj_length); + CHECK_LT(offset, buffer.length()); const void* ptr; if (is_forward) { - ptr = memchr(ts_obj_data + offset, needle, ts_obj_length - offset); + ptr = memchr(buffer.data() + offset, needle, buffer.length() - offset); } else { - ptr = node::stringsearch::MemrchrFill(ts_obj_data, needle, offset + 1); + ptr = node::stringsearch::MemrchrFill(buffer.data(), needle, offset + 1); } const char* ptr_char = static_cast(ptr); - args.GetReturnValue().Set(ptr ? static_cast(ptr_char - ts_obj_data) + args.GetReturnValue().Set(ptr ? static_cast(ptr_char - buffer.data()) : -1); } diff --git a/src/node_config.cc b/src/node_config.cc index 335ebedc583bdc..92985dff2f8b0c 100644 --- a/src/node_config.cc +++ b/src/node_config.cc @@ -1,4 +1,5 @@ #include "env-inl.h" +#include "memory_tracker.h" #include "node.h" #include "node_i18n.h" #include "node_native_module_env.h" diff --git a/src/node_constants.cc b/src/node_constants.cc index aa2afc59d95e64..7c9e4ce276112b 100644 --- a/src/node_constants.cc +++ b/src/node_constants.cc @@ -19,6 +19,7 @@ // OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE // USE OR OTHER DEALINGS IN THE SOFTWARE. +#include "env-inl.h" #include "node_constants.h" #include "node_internals.h" #include "util-inl.h" diff --git a/src/node_contextify.cc b/src/node_contextify.cc index 77622f8cbd6d2b..ee5a9db574f0f9 100644 --- a/src/node_contextify.cc +++ b/src/node_contextify.cc @@ -21,6 +21,7 @@ #include "node_contextify.h" +#include "memory_tracker-inl.h" #include "node_internals.h" #include "node_watchdog.h" #include "base_object-inl.h" diff --git a/src/node_credentials.cc b/src/node_credentials.cc index d384504f2ac906..6f99cf6641eec3 100644 --- a/src/node_credentials.cc +++ b/src/node_credentials.cc @@ -1,3 +1,4 @@ +#include "env-inl.h" #include "node_internals.h" #include "util-inl.h" diff --git a/src/node_crypto.cc b/src/node_crypto.cc index bbb07e5b48b4a2..a5710dc33b62b9 100644 --- a/src/node_crypto.cc +++ b/src/node_crypto.cc @@ -32,7 +32,9 @@ #include "async_wrap-inl.h" #include "base_object-inl.h" #include "env-inl.h" +#include "memory_tracker-inl.h" #include "string_bytes.h" +#include "threadpoolwork-inl.h" #include "util-inl.h" #include "v8.h" @@ -5764,11 +5766,10 @@ void ECDH::SetPrivateKey(const FunctionCallbackInfo& args) { ASSIGN_OR_RETURN_UNWRAP(&ecdh, args.Holder()); THROW_AND_RETURN_IF_NOT_BUFFER(env, args[0], "Private key"); + ArrayBufferViewContents priv_buffer(args[0]); BignumPointer priv(BN_bin2bn( - reinterpret_cast(Buffer::Data(args[0].As())), - Buffer::Length(args[0].As()), - nullptr)); + priv_buffer.data(), priv_buffer.length(), nullptr)); if (!priv) return env->ThrowError("Failed to convert Buffer to BN"); @@ -6685,14 +6686,11 @@ OpenSSLBuffer ExportChallenge(const char* data, int len) { void ExportChallenge(const FunctionCallbackInfo& args) { Environment* env = Environment::GetCurrent(args); - size_t len = Buffer::Length(args[0]); - if (len == 0) + ArrayBufferViewContents input(args[0]); + if (input.length() == 0) return args.GetReturnValue().SetEmptyString(); - char* data = Buffer::Data(args[0]); - CHECK_NOT_NULL(data); - - OpenSSLBuffer cert = ExportChallenge(data, len); + OpenSSLBuffer cert = ExportChallenge(input.data(), input.length()); if (!cert) return args.GetReturnValue().SetEmptyString(); @@ -6747,16 +6745,13 @@ void ConvertKey(const FunctionCallbackInfo& args) { void TimingSafeEqual(const FunctionCallbackInfo& args) { - CHECK(Buffer::HasInstance(args[0])); - CHECK(Buffer::HasInstance(args[1])); - - size_t buf_length = Buffer::Length(args[0]); - CHECK_EQ(buf_length, Buffer::Length(args[1])); + ArrayBufferViewContents buf1(args[0]); + ArrayBufferViewContents buf2(args[1]); - const char* buf1 = Buffer::Data(args[0]); - const char* buf2 = Buffer::Data(args[1]); + CHECK_EQ(buf1.length(), buf2.length()); - return args.GetReturnValue().Set(CRYPTO_memcmp(buf1, buf2, buf_length) == 0); + return args.GetReturnValue().Set( + CRYPTO_memcmp(buf1.data(), buf2.data(), buf1.length()) == 0); } void InitCryptoOnce() { diff --git a/src/node_crypto_bio.cc b/src/node_crypto_bio.cc index fcbe30ca9c07c0..9f06801c3ae20c 100644 --- a/src/node_crypto_bio.cc +++ b/src/node_crypto_bio.cc @@ -20,6 +20,7 @@ // USE OR OTHER DEALINGS IN THE SOFTWARE. #include "base_object-inl.h" +#include "memory_tracker-inl.h" #include "node_crypto_bio.h" #include "openssl/bio.h" #include "util-inl.h" diff --git a/src/node_file.cc b/src/node_file.cc index 4da2a4f53cb581..8f82d639d6d0a3 100644 --- a/src/node_file.cc +++ b/src/node_file.cc @@ -21,6 +21,7 @@ #include "node_file.h" #include "aliased_buffer.h" +#include "memory_tracker-inl.h" #include "node_buffer.h" #include "node_process.h" #include "node_stat_watcher.h" diff --git a/src/node_http2.cc b/src/node_http2.cc index 0c650290c2b7f2..3a7591f31afda7 100644 --- a/src/node_http2.cc +++ b/src/node_http2.cc @@ -1,5 +1,6 @@ #include "aliased_buffer.h" #include "debug_utils.h" +#include "memory_tracker-inl.h" #include "node.h" #include "node_buffer.h" #include "node_http2.h" @@ -1781,11 +1782,13 @@ void Http2Session::OnStreamRead(ssize_t nread, const uv_buf_t& buf_) { // Shrink to the actual amount of used data. buf.Resize(nread); - IncrementCurrentSessionMemory(buf.size()); + IncrementCurrentSessionMemory(nread); OnScopeLeave on_scope_leave([&]() { // Once finished handling this write, reset the stream buffer. // The memory has either been free()d or was handed over to V8. - DecrementCurrentSessionMemory(buf.size()); + // We use `nread` instead of `buf.size()` here, because the buffer is + // cleared as part of the `.ToArrayBuffer()` call below. + DecrementCurrentSessionMemory(nread); stream_buf_ab_ = Local(); stream_buf_ = uv_buf_init(nullptr, 0); }); diff --git a/src/node_http_parser_impl.h b/src/node_http_parser_impl.h index a354c6fcc51eba..1afedb0509b4aa 100644 --- a/src/node_http_parser_impl.h +++ b/src/node_http_parser_impl.h @@ -466,18 +466,15 @@ class Parser : public AsyncWrap, public StreamListener { CHECK(parser->current_buffer_.IsEmpty()); CHECK_EQ(parser->current_buffer_len_, 0); CHECK_NULL(parser->current_buffer_data_); - CHECK_EQ(Buffer::HasInstance(args[0]), true); - Local buffer_obj = args[0].As(); - char* buffer_data = Buffer::Data(buffer_obj); - size_t buffer_len = Buffer::Length(buffer_obj); + ArrayBufferViewContents buffer(args[0]); // This is a hack to get the current_buffer to the callbacks with the least // amount of overhead. Nothing else will run while http_parser_execute() // runs, therefore this pointer can be set and used for the execution. - parser->current_buffer_ = buffer_obj; + parser->current_buffer_ = args[0].As(); - Local ret = parser->Execute(buffer_data, buffer_len); + Local ret = parser->Execute(buffer.data(), buffer.length()); if (!ret.IsEmpty()) args.GetReturnValue().Set(ret); @@ -643,7 +640,7 @@ class Parser : public AsyncWrap, public StreamListener { } - Local Execute(char* data, size_t len) { + Local Execute(const char* data, size_t len) { EscapableHandleScope scope(env()->isolate()); current_buffer_len_ = len; @@ -857,7 +854,7 @@ class Parser : public AsyncWrap, public StreamListener { bool got_exception_; Local current_buffer_; size_t current_buffer_len_; - char* current_buffer_data_; + const char* current_buffer_data_; #ifdef NODE_EXPERIMENTAL_HTTP unsigned int execute_depth_ = 0; bool pending_pause_ = false; diff --git a/src/node_http_parser_llhttp.cc b/src/node_http_parser_llhttp.cc index d2063873f17efd..6e1d18d3af94ff 100644 --- a/src/node_http_parser_llhttp.cc +++ b/src/node_http_parser_llhttp.cc @@ -1,6 +1,7 @@ #define NODE_EXPERIMENTAL_HTTP 1 #include "node_http_parser_impl.h" +#include "memory_tracker-inl.h" #include "node_metadata.h" #include "util-inl.h" diff --git a/src/node_http_parser_traditional.cc b/src/node_http_parser_traditional.cc index 7b413af8b6ce0c..11aa387f109828 100644 --- a/src/node_http_parser_traditional.cc +++ b/src/node_http_parser_traditional.cc @@ -2,6 +2,7 @@ #undef NODE_EXPERIMENTAL_HTTP #endif +#include "memory_tracker-inl.h" #include "node_http_parser_impl.h" #include "node_metadata.h" #include "util-inl.h" diff --git a/src/node_i18n.cc b/src/node_i18n.cc index 0c565c447fb862..ad5c8283924afa 100644 --- a/src/node_i18n.cc +++ b/src/node_i18n.cc @@ -45,7 +45,6 @@ #if defined(NODE_HAVE_I18N_SUPPORT) #include "base_object-inl.h" -#include "env-inl.h" #include "node.h" #include "node_buffer.h" #include "node_errors.h" @@ -206,14 +205,13 @@ class ConverterObject : public BaseObject, Converter { ConverterObject* converter; ASSIGN_OR_RETURN_UNWRAP(&converter, args[0].As()); - SPREAD_BUFFER_ARG(args[1], input_obj); + ArrayBufferViewContents input(args[1]); int flags = args[2]->Uint32Value(env->context()).ToChecked(); UErrorCode status = U_ZERO_ERROR; MaybeStackBuffer result; MaybeLocal ret; - size_t limit = ucnv_getMinCharSize(converter->conv) * - input_obj_length; + size_t limit = ucnv_getMinCharSize(converter->conv) * input.length(); if (limit > 0) result.AllocateSufficientStorage(limit); @@ -226,8 +224,8 @@ class ConverterObject : public BaseObject, Converter { } }); - const char* source = input_obj_data; - size_t source_length = input_obj_length; + const char* source = input.data(); + size_t source_length = input.length(); if (converter->unicode_ && !converter->ignoreBOM_ && !converter->bomSeen_) { int32_t bomOffset = 0; @@ -456,8 +454,7 @@ void Transcode(const FunctionCallbackInfo&args) { UErrorCode status = U_ZERO_ERROR; MaybeLocal result; - CHECK(Buffer::HasInstance(args[0])); - SPREAD_BUFFER_ARG(args[0], ts_obj); + ArrayBufferViewContents input(args[0]); const enum encoding fromEncoding = ParseEncoding(isolate, args[1], BUFFER); const enum encoding toEncoding = ParseEncoding(isolate, args[2], BUFFER); @@ -491,7 +488,7 @@ void Transcode(const FunctionCallbackInfo&args) { } result = tfn(env, EncodingName(fromEncoding), EncodingName(toEncoding), - ts_obj_data, ts_obj_length, &status); + input.data(), input.length(), &status); } else { status = U_ILLEGAL_ARGUMENT_ERROR; } diff --git a/src/node_internals.h b/src/node_internals.h index 91cc0efd508d8c..21625e60232bbe 100644 --- a/src/node_internals.h +++ b/src/node_internals.h @@ -24,7 +24,7 @@ #if defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS -#include "env-inl.h" +#include "env.h" #include "node.h" #include "node_binding.h" #include "node_mutex.h" @@ -255,27 +255,6 @@ class ThreadPoolWork { uv_work_t work_req_; }; -void ThreadPoolWork::ScheduleWork() { - env_->IncreaseWaitingRequestCounter(); - int status = uv_queue_work( - env_->event_loop(), - &work_req_, - [](uv_work_t* req) { - ThreadPoolWork* self = ContainerOf(&ThreadPoolWork::work_req_, req); - self->DoThreadPoolWork(); - }, - [](uv_work_t* req, int status) { - ThreadPoolWork* self = ContainerOf(&ThreadPoolWork::work_req_, req); - self->env_->DecreaseWaitingRequestCounter(); - self->AfterThreadPoolWork(status); - }); - CHECK_EQ(status, 0); -} - -int ThreadPoolWork::CancelWork() { - return uv_cancel(reinterpret_cast(&work_req_)); -} - #define TRACING_CATEGORY_NODE "node" #define TRACING_CATEGORY_NODE1(one) \ TRACING_CATEGORY_NODE "," \ @@ -345,17 +324,15 @@ class DiagnosticFilename { public: static void LocalTime(TIME_TYPE* tm_struct); - DiagnosticFilename(Environment* env, - const char* prefix, - const char* ext) : - filename_(MakeFilename(env->thread_id(), prefix, ext)) {} + inline DiagnosticFilename(Environment* env, + const char* prefix, + const char* ext); - DiagnosticFilename(uint64_t thread_id, - const char* prefix, - const char* ext) : - filename_(MakeFilename(thread_id, prefix, ext)) {} + inline DiagnosticFilename(uint64_t thread_id, + const char* prefix, + const char* ext); - const char* operator*() const { return filename_.c_str(); } + inline const char* operator*() const; private: static std::string MakeFilename( diff --git a/src/node_messaging.cc b/src/node_messaging.cc index ea7b48779ca188..fa583a2570315b 100644 --- a/src/node_messaging.cc +++ b/src/node_messaging.cc @@ -2,6 +2,7 @@ #include "async_wrap-inl.h" #include "debug_utils.h" +#include "memory_tracker-inl.h" #include "node_contextify.h" #include "node_buffer.h" #include "node_errors.h" diff --git a/src/node_native_module.h b/src/node_native_module.h index 5450c63c161cf2..fabaea75686161 100644 --- a/src/node_native_module.h +++ b/src/node_native_module.h @@ -11,6 +11,9 @@ #include "node_union_bytes.h" #include "v8.h" +// Forward declare test fixture for `friend` declaration. +class PerProcessTest; + namespace node { namespace native_module { @@ -82,6 +85,8 @@ class NativeModuleLoader { // Used to synchronize access to the code cache map Mutex code_cache_mutex_; + + friend class ::PerProcessTest; }; } // namespace native_module diff --git a/src/node_options.cc b/src/node_options.cc index 12e0cb09d085fe..35094d66323d55 100644 --- a/src/node_options.cc +++ b/src/node_options.cc @@ -117,11 +117,6 @@ void EnvironmentOptions::CheckOptions(std::vector* errors) { } } - if (experimental_json_modules && !experimental_modules) { - errors->push_back("--experimental-json-modules requires " - "--experimental-modules be enabled"); - } - if (experimental_wasm_modules && !experimental_modules) { errors->push_back("--experimental-wasm-modules requires " "--experimental-modules be enabled"); @@ -173,6 +168,19 @@ void EnvironmentOptions::CheckOptions(std::vector* errors) { } } + if (!heap_prof) { + if (!heap_prof_name.empty()) { + errors->push_back("--heap-prof-name must be used with --heap-prof"); + } + if (!heap_prof_dir.empty()) { + errors->push_back("--heap-prof-dir must be used with --heap-prof"); + } + // We can't catch the case where the value passed is the default value, + // then the option just becomes a noop which is fine. + if (heap_prof_interval != kDefaultHeapProfInterval) { + errors->push_back("--heap-prof-interval must be used with --heap-prof"); + } + } debug_options_.CheckOptions(errors); #endif // HAVE_INSPECTOR } @@ -271,10 +279,6 @@ DebugOptionsParser::DebugOptionsParser() { } EnvironmentOptionsParser::EnvironmentOptionsParser() { - AddOption("--experimental-json-modules", - "experimental JSON interop support for the ES Module loader", - &EnvironmentOptions::experimental_json_modules, - kAllowedInEnvironment); AddOption("--experimental-modules", "experimental ES Module support and caching modules", &EnvironmentOptions::experimental_modules, @@ -317,6 +321,11 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() { "(default: llhttp).", &EnvironmentOptions::http_parser, kAllowedInEnvironment); + AddOption("--http-server-default-timeout", + "Default http server socket timeout in ms " + "(default: 120000)", + &EnvironmentOptions::http_server_default_timeout, + kAllowedInEnvironment); AddOption("--input-type", "set module type for string input", &EnvironmentOptions::module_type, @@ -378,6 +387,24 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() { "Directory where the V8 profiles generated by --cpu-prof will be " "placed. Does not affect --prof.", &EnvironmentOptions::cpu_prof_dir); + AddOption( + "--heap-prof", + "Start the V8 heap profiler on start up, and write the heap profile " + "to disk before exit. If --heap-prof-dir is not specified, write " + "the profile to the current working directory.", + &EnvironmentOptions::heap_prof); + AddOption("--heap-prof-name", + "specified file name of the V8 CPU profile generated with " + "--heap-prof", + &EnvironmentOptions::heap_prof_name); + AddOption("--heap-prof-dir", + "Directory where the V8 heap profiles generated by --heap-prof " + "will be placed.", + &EnvironmentOptions::heap_prof_dir); + AddOption("--heap-prof-interval", + "specified sampling interval in bytes for the V8 heap " + "profile generated with --heap-prof. (default: 512 * 1024)", + &EnvironmentOptions::heap_prof_interval); #endif // HAVE_INSPECTOR AddOption("--redirect-warnings", "write warnings to file instead of stderr", diff --git a/src/node_options.h b/src/node_options.h index b0a1844df58d2e..174f5369854a6a 100644 --- a/src/node_options.h +++ b/src/node_options.h @@ -91,7 +91,6 @@ class DebugOptions : public Options { class EnvironmentOptions : public Options { public: bool abort_on_uncaught_exception = false; - bool experimental_json_modules = false; bool experimental_modules = false; std::string es_module_specifier_resolution; bool experimental_wasm_modules = false; @@ -103,6 +102,7 @@ class EnvironmentOptions : public Options { bool frozen_intrinsics = false; std::string heap_snapshot_signal; std::string http_parser = "llhttp"; + uint64_t http_server_default_timeout = 120000; bool no_deprecation = false; bool no_force_async_hooks_checks = false; bool no_warnings = false; @@ -116,6 +116,11 @@ class EnvironmentOptions : public Options { uint64_t cpu_prof_interval = kDefaultCpuProfInterval; std::string cpu_prof_name; bool cpu_prof = false; + std::string heap_prof_dir; + std::string heap_prof_name; + static const uint64_t kDefaultHeapProfInterval = 512 * 1024; + uint64_t heap_prof_interval = kDefaultHeapProfInterval; + bool heap_prof = false; #endif // HAVE_INSPECTOR std::string redirect_warnings; bool throw_deprecation = false; diff --git a/src/node_os.cc b/src/node_os.cc index d2387f2dc96bf7..b6fb305948e234 100644 --- a/src/node_os.cc +++ b/src/node_os.cc @@ -49,6 +49,7 @@ using v8::Integer; using v8::Isolate; using v8::Local; using v8::MaybeLocal; +using v8::NewStringType; using v8::Null; using v8::Number; using v8::Object; @@ -69,7 +70,9 @@ static void GetHostname(const FunctionCallbackInfo& args) { return args.GetReturnValue().SetUndefined(); } - args.GetReturnValue().Set(OneByteString(env->isolate(), buf)); + args.GetReturnValue().Set( + String::NewFromUtf8(env->isolate(), buf, NewStringType::kNormal) + .ToLocalChecked()); } @@ -84,7 +87,9 @@ static void GetOSType(const FunctionCallbackInfo& args) { return args.GetReturnValue().SetUndefined(); } - args.GetReturnValue().Set(OneByteString(env->isolate(), info.sysname)); + args.GetReturnValue().Set( + String::NewFromUtf8(env->isolate(), info.sysname, NewStringType::kNormal) + .ToLocalChecked()); } @@ -99,7 +104,9 @@ static void GetOSRelease(const FunctionCallbackInfo& args) { return args.GetReturnValue().SetUndefined(); } - args.GetReturnValue().Set(OneByteString(env->isolate(), info.release)); + args.GetReturnValue().Set( + String::NewFromUtf8(env->isolate(), info.release, NewStringType::kNormal) + .ToLocalChecked()); } diff --git a/src/node_perf.cc b/src/node_perf.cc index 08632020300038..f43c10213624bb 100644 --- a/src/node_perf.cc +++ b/src/node_perf.cc @@ -1,4 +1,5 @@ #include "aliased_buffer.h" +#include "memory_tracker-inl.h" #include "node_internals.h" #include "node_perf.h" #include "node_buffer.h" diff --git a/src/node_platform.cc b/src/node_platform.cc index 406146b841e25e..b2e8d77ec7a987 100644 --- a/src/node_platform.cc +++ b/src/node_platform.cc @@ -445,17 +445,6 @@ NodePlatform::ForIsolate(Isolate* isolate) { return data; } -void NodePlatform::CallOnForegroundThread(Isolate* isolate, Task* task) { - ForIsolate(isolate)->PostTask(std::unique_ptr(task)); -} - -void NodePlatform::CallDelayedOnForegroundThread(Isolate* isolate, - Task* task, - double delay_in_seconds) { - ForIsolate(isolate)->PostDelayedTask( - std::unique_ptr(task), delay_in_seconds); -} - bool NodePlatform::FlushForegroundTasks(Isolate* isolate) { return ForIsolate(isolate)->FlushForegroundTasksInternal(); } diff --git a/src/node_platform.h b/src/node_platform.h index 66f86cfa7e74d5..d2eb325c12113e 100644 --- a/src/node_platform.h +++ b/src/node_platform.h @@ -145,9 +145,14 @@ class NodePlatform : public MultiIsolatePlatform { void CallOnWorkerThread(std::unique_ptr task) override; void CallDelayedOnWorkerThread(std::unique_ptr task, double delay_in_seconds) override; - void CallOnForegroundThread(v8::Isolate* isolate, v8::Task* task) override; - void CallDelayedOnForegroundThread(v8::Isolate* isolate, v8::Task* task, - double delay_in_seconds) override; + void CallOnForegroundThread(v8::Isolate* isolate, v8::Task* task) override { + UNREACHABLE(); + } + void CallDelayedOnForegroundThread(v8::Isolate* isolate, + v8::Task* task, + double delay_in_seconds) override { + UNREACHABLE(); + } bool IdleTasksEnabled(v8::Isolate* isolate) override; double MonotonicallyIncreasingTime() override; double CurrentClockTimeMillis() override; diff --git a/src/node_report.cc b/src/node_report.cc index 578da4376e07da..18607656256d51 100644 --- a/src/node_report.cc +++ b/src/node_report.cc @@ -1,5 +1,7 @@ +#include "env-inl.h" #include "node_report.h" #include "debug_utils.h" +#include "diagnosticfilename-inl.h" #include "node_internals.h" #include "node_metadata.h" #include "util.h" diff --git a/src/node_serdes.cc b/src/node_serdes.cc index 41ee8afd8cbcc5..a2d185c4167a75 100644 --- a/src/node_serdes.cc +++ b/src/node_serdes.cc @@ -274,8 +274,8 @@ void SerializerContext::WriteRawBytes(const FunctionCallbackInfo& args) { ctx->env(), "source must be a TypedArray or a DataView"); } - ctx->serializer_.WriteRawBytes(Buffer::Data(args[0]), - Buffer::Length(args[0])); + ArrayBufferViewContents bytes(args[0]); + ctx->serializer_.WriteRawBytes(bytes.data(), bytes.length()); } DeserializerContext::DeserializerContext(Environment* env, diff --git a/src/node_stat_watcher.cc b/src/node_stat_watcher.cc index b7fb45900f9bb8..ae30825cbbdbd2 100644 --- a/src/node_stat_watcher.cc +++ b/src/node_stat_watcher.cc @@ -19,6 +19,7 @@ // OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE // USE OR OTHER DEALINGS IN THE SOFTWARE. +#include "memory_tracker-inl.h" #include "node_stat_watcher.h" #include "async_wrap-inl.h" #include "env.h" diff --git a/src/node_trace_events.cc b/src/node_trace_events.cc index 13072a3340ced9..f5852076b4ec68 100644 --- a/src/node_trace_events.cc +++ b/src/node_trace_events.cc @@ -1,5 +1,6 @@ #include "base_object-inl.h" -#include "env.h" +#include "env-inl.h" +#include "memory_tracker-inl.h" #include "node.h" #include "node_internals.h" #include "node_v8_platform-inl.h" diff --git a/src/node_url.h b/src/node_url.h index e85b14e2bdf35c..e85ca6e7129f6a 100644 --- a/src/node_url.h +++ b/src/node_url.h @@ -4,7 +4,7 @@ #if defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS #include "node.h" -#include "env-inl.h" +#include "env.h" #include diff --git a/src/node_v8.cc b/src/node_v8.cc index 5adc53b84d87c4..1227ebec5362e5 100644 --- a/src/node_v8.cc +++ b/src/node_v8.cc @@ -52,7 +52,9 @@ using v8::Value; V(5, heap_size_limit, kHeapSizeLimitIndex) \ V(6, malloced_memory, kMallocedMemoryIndex) \ V(7, peak_malloced_memory, kPeakMallocedMemoryIndex) \ - V(8, does_zap_garbage, kDoesZapGarbageIndex) + V(8, does_zap_garbage, kDoesZapGarbageIndex) \ + V(9, number_of_native_contexts, kNumberOfNativeContextsIndex) \ + V(10, number_of_detached_contexts, kNumberOfDetachedContextsIndex) #define V(a, b, c) +1 static const size_t kHeapStatisticsPropertiesCount = diff --git a/src/node_version.h b/src/node_version.h index 52d038a902e748..1c04b102721f89 100644 --- a/src/node_version.h +++ b/src/node_version.h @@ -23,13 +23,13 @@ #define SRC_NODE_VERSION_H_ #define NODE_MAJOR_VERSION 12 -#define NODE_MINOR_VERSION 3 -#define NODE_PATCH_VERSION 2 +#define NODE_MINOR_VERSION 4 +#define NODE_PATCH_VERSION 0 #define NODE_VERSION_IS_LTS 0 #define NODE_VERSION_LTS_CODENAME "" -#define NODE_VERSION_IS_RELEASE 0 +#define NODE_VERSION_IS_RELEASE 1 #ifndef NODE_STRINGIFY #define NODE_STRINGIFY(n) NODE_STRINGIFY_HELPER(n) diff --git a/src/node_watchdog.cc b/src/node_watchdog.cc index 7c62aafa82257b..0c055489fcff68 100644 --- a/src/node_watchdog.cc +++ b/src/node_watchdog.cc @@ -19,11 +19,13 @@ // OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE // USE OR OTHER DEALINGS IN THE SOFTWARE. -#include "node_watchdog.h" #include + #include "debug_utils.h" +#include "env-inl.h" #include "node_errors.h" #include "node_internals.h" +#include "node_watchdog.h" #include "util-inl.h" namespace node { diff --git a/src/node_worker.cc b/src/node_worker.cc index 20b883664da207..e84b36f132ae3b 100644 --- a/src/node_worker.cc +++ b/src/node_worker.cc @@ -1,5 +1,6 @@ #include "node_worker.h" #include "debug_utils.h" +#include "memory_tracker-inl.h" #include "node_errors.h" #include "node_buffer.h" #include "node_options-inl.h" diff --git a/src/node_zlib.cc b/src/node_zlib.cc index f389257882215e..30fef0ff1d4d57 100644 --- a/src/node_zlib.cc +++ b/src/node_zlib.cc @@ -19,11 +19,13 @@ // OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE // USE OR OTHER DEALINGS IN THE SOFTWARE. +#include "memory_tracker-inl.h" #include "node.h" #include "node_buffer.h" #include "async_wrap-inl.h" #include "env-inl.h" +#include "threadpoolwork-inl.h" #include "util-inl.h" #include "v8.h" diff --git a/src/threadpoolwork-inl.h b/src/threadpoolwork-inl.h new file mode 100644 index 00000000000000..8bba988b18db0f --- /dev/null +++ b/src/threadpoolwork-inl.h @@ -0,0 +1,57 @@ +// Copyright Joyent, Inc. and other Node contributors. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + +#ifndef SRC_THREADPOOLWORK_INL_H_ +#define SRC_THREADPOOLWORK_INL_H_ + +#if defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS + +#include "util-inl.h" +#include "node_internals.h" + +namespace node { + +void ThreadPoolWork::ScheduleWork() { + env_->IncreaseWaitingRequestCounter(); + int status = uv_queue_work( + env_->event_loop(), + &work_req_, + [](uv_work_t* req) { + ThreadPoolWork* self = ContainerOf(&ThreadPoolWork::work_req_, req); + self->DoThreadPoolWork(); + }, + [](uv_work_t* req, int status) { + ThreadPoolWork* self = ContainerOf(&ThreadPoolWork::work_req_, req); + self->env_->DecreaseWaitingRequestCounter(); + self->AfterThreadPoolWork(status); + }); + CHECK_EQ(status, 0); +} + +int ThreadPoolWork::CancelWork() { + return uv_cancel(reinterpret_cast(&work_req_)); +} + +} // namespace node + +#endif // defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS + +#endif // SRC_THREADPOOLWORK_INL_H_ diff --git a/src/tls_wrap.cc b/src/tls_wrap.cc index cd6321b9693c3b..17329c4b5ce56a 100644 --- a/src/tls_wrap.cc +++ b/src/tls_wrap.cc @@ -22,6 +22,7 @@ #include "tls_wrap.h" #include "async_wrap-inl.h" #include "debug_utils.h" +#include "memory_tracker-inl.h" #include "node_buffer.h" // Buffer #include "node_crypto.h" // SecureContext #include "node_crypto_bio.h" // NodeBIO @@ -186,9 +187,9 @@ void TLSWrap::Receive(const FunctionCallbackInfo& args) { TLSWrap* wrap; ASSIGN_OR_RETURN_UNWRAP(&wrap, args.Holder()); - CHECK(Buffer::HasInstance(args[0])); - char* data = Buffer::Data(args[0]); - size_t len = Buffer::Length(args[0]); + ArrayBufferViewContents buffer(args[0]); + const char* data = buffer.data(); + size_t len = buffer.length(); Debug(wrap, "Receiving %zu bytes injected from JS", len); // Copy given buffer entirely or partiall if handle becomes closed @@ -301,7 +302,7 @@ void TLSWrap::EncOut() { // No encrypted output ready to write to the underlying stream. if (BIO_pending(enc_out_) == 0) { Debug(this, "No pending encrypted output"); - if (pending_cleartext_input_.empty()) { + if (pending_cleartext_input_.size() == 0) { if (!in_dowrite_) { Debug(this, "No pending cleartext input, not inside DoWrite()"); InvokeQueued(0); @@ -572,28 +573,21 @@ void TLSWrap::ClearIn() { return; } - std::vector buffers; - buffers.swap(pending_cleartext_input_); + if (pending_cleartext_input_.size() == 0) { + Debug(this, "Returning from ClearIn(), no pending data"); + return; + } + AllocatedBuffer data = std::move(pending_cleartext_input_); crypto::MarkPopErrorOnReturn mark_pop_error_on_return; - size_t i; - int written = 0; - for (i = 0; i < buffers.size(); ++i) { - size_t avail = buffers[i].len; - char* data = buffers[i].base; - written = SSL_write(ssl_.get(), data, avail); - Debug(this, "Writing %zu bytes, written = %d", avail, written); - CHECK(written == -1 || written == static_cast(avail)); - if (written == -1) - break; - } + int written = SSL_write(ssl_.get(), data.data(), data.size()); + Debug(this, "Writing %zu bytes, written = %d", data.size(), written); + CHECK(written == -1 || written == static_cast(data.size())); // All written - if (i == buffers.size()) { + if (written != -1) { Debug(this, "Successfully wrote all data to SSL"); - // We wrote all the buffers, so no writes failed (written < 0 on failure). - CHECK_GE(written, 0); return; } @@ -611,13 +605,10 @@ void TLSWrap::ClearIn() { // .code/.function/.etc, if possible. InvokeQueued(UV_EPROTO, error_str.c_str()); } else { - Debug(this, "Pushing back %zu buffers", buffers.size() - i); - // Push back the not-yet-written pending buffers into their queue. - // This can be skipped in the error case because no further writes - // would succeed anyway. - pending_cleartext_input_.insert(pending_cleartext_input_.end(), - buffers.begin() + i, - buffers.end()); + Debug(this, "Pushing data back"); + // Push back the not-yet-written data. This can be skipped in the error + // case because no further writes would succeed anyway. + pending_cleartext_input_ = std::move(data); } return; @@ -704,14 +695,10 @@ int TLSWrap::DoWrite(WriteWrap* w, return UV_EPROTO; } - bool empty = true; + size_t length = 0; size_t i; - for (i = 0; i < count; i++) { - if (bufs[i].len > 0) { - empty = false; - break; - } - } + for (i = 0; i < count; i++) + length += bufs[i].len; // We want to trigger a Write() on the underlying stream to drive the stream // system, but don't want to encrypt empty buffers into a TLS frame, so see @@ -723,7 +710,7 @@ int TLSWrap::DoWrite(WriteWrap* w, // stream. Since the bufs are empty, it won't actually write non-TLS data // onto the socket, we just want the side-effects. After, make sure the // WriteWrap was accepted by the stream, or that we call Done() on it. - if (empty) { + if (length == 0) { Debug(this, "Empty write"); ClearOut(); if (BIO_pending(enc_out_) == 0) { @@ -747,23 +734,36 @@ int TLSWrap::DoWrite(WriteWrap* w, current_write_ = w; // Write encrypted data to underlying stream and call Done(). - if (empty) { + if (length == 0) { EncOut(); return 0; } + AllocatedBuffer data; crypto::MarkPopErrorOnReturn mark_pop_error_on_return; int written = 0; - for (i = 0; i < count; i++) { - written = SSL_write(ssl_.get(), bufs[i].base, bufs[i].len); - CHECK(written == -1 || written == static_cast(bufs[i].len)); - Debug(this, "Writing %zu bytes, written = %d", bufs[i].len, written); - if (written == -1) - break; + if (count != 1) { + data = env()->AllocateManaged(length); + size_t offset = 0; + for (i = 0; i < count; i++) { + memcpy(data.data() + offset, bufs[i].base, bufs[i].len); + offset += bufs[i].len; + } + written = SSL_write(ssl_.get(), data.data(), length); + } else { + // Only one buffer: try to write directly, only store if it fails + written = SSL_write(ssl_.get(), bufs[0].base, bufs[0].len); + if (written == -1) { + data = env()->AllocateManaged(length); + memcpy(data.data(), bufs[0].base, bufs[0].len); + } } - if (i != count) { + CHECK(written == -1 || written == static_cast(length)); + Debug(this, "Writing %zu bytes, written = %d", length, written); + + if (written == -1) { int err; Local arg = GetSSLError(written, &err, &error_); @@ -774,11 +774,10 @@ int TLSWrap::DoWrite(WriteWrap* w, return UV_EPROTO; } - Debug(this, "Saving %zu buffers for later write", count - i); + Debug(this, "Saving data for later write"); // Otherwise, save unwritten data so it can be written later by ClearIn(). - pending_cleartext_input_.insert(pending_cleartext_input_.end(), - &bufs[i], - &bufs[count]); + CHECK_EQ(pending_cleartext_input_.size(), 0); + pending_cleartext_input_ = std::move(data); } // Write any encrypted/handshake output that may be ready. @@ -938,9 +937,19 @@ void TLSWrap::EnableTrace( #if HAVE_SSL_TRACE if (wrap->ssl_) { - BIO* b = BIO_new_fp(stderr, BIO_NOCLOSE | BIO_FP_TEXT); - SSL_set_msg_callback(wrap->ssl_.get(), SSL_trace); - SSL_set_msg_callback_arg(wrap->ssl_.get(), b); + wrap->bio_trace_.reset(BIO_new_fp(stderr, BIO_NOCLOSE | BIO_FP_TEXT)); + SSL_set_msg_callback(wrap->ssl_.get(), [](int write_p, int version, int + content_type, const void* buf, size_t len, SSL* ssl, void* arg) + -> void { + // BIO_write(), etc., called by SSL_trace, may error. The error should + // be ignored, trace is a "best effort", and its usually because stderr + // is a non-blocking pipe, and its buffer has overflowed. Leaving errors + // on the stack that can get picked up by later SSL_ calls causes + // unwanted failures in SSL_ calls, so keep the error stack unchanged. + crypto::MarkPopErrorOnReturn mark_pop_error_on_return; + SSL_trace(write_p, version, content_type, buf, len, ssl, arg); + }); + SSL_set_msg_callback_arg(wrap->ssl_.get(), wrap->bio_trace_.get()); } #endif } @@ -1033,6 +1042,14 @@ int TLSWrap::SelectSNIContextCallback(SSL* s, int* ad, void* arg) { Local object = p->object(); Local ctx; + // Set the servername as early as possible + Local owner = p->GetOwner(); + if (!owner->Set(env->context(), + env->servername_string(), + OneByteString(env->isolate(), servername)).FromMaybe(false)) { + return SSL_TLSEXT_ERR_NOACK; + } + if (!object->Get(env->context(), env->sni_context_string()).ToLocal(&ctx)) return SSL_TLSEXT_ERR_NOACK; @@ -1073,7 +1090,9 @@ void TLSWrap::GetWriteQueueSize(const FunctionCallbackInfo& info) { void TLSWrap::MemoryInfo(MemoryTracker* tracker) const { tracker->TrackField("error", error_); - tracker->TrackField("pending_cleartext_input", pending_cleartext_input_); + tracker->TrackFieldWithSize("pending_cleartext_input", + pending_cleartext_input_.size(), + "AllocatedBuffer"); if (enc_in_ != nullptr) tracker->TrackField("enc_in", crypto::NodeBIO::FromBIO(enc_in_)); if (enc_out_ != nullptr) diff --git a/src/tls_wrap.h b/src/tls_wrap.h index b866bbb7af122e..631ef8e7c3d8db 100644 --- a/src/tls_wrap.h +++ b/src/tls_wrap.h @@ -174,7 +174,7 @@ class TLSWrap : public AsyncWrap, BIO* enc_in_ = nullptr; // StreamListener fills this for SSL_read(). BIO* enc_out_ = nullptr; // SSL_write()/handshake fills this for EncOut(). // Waiting for ClearIn() to pass to SSL_write(). - std::vector pending_cleartext_input_; + AllocatedBuffer pending_cleartext_input_; size_t write_size_ = 0; WriteWrap* current_write_ = nullptr; bool in_dowrite_ = false; @@ -193,6 +193,8 @@ class TLSWrap : public AsyncWrap, private: static void GetWriteQueueSize( const v8::FunctionCallbackInfo& info); + + crypto::BIOPointer bio_trace_; }; } // namespace node diff --git a/src/util-inl.h b/src/util-inl.h index 1abebf6e1374e8..5f8e7acb2a8e52 100644 --- a/src/util-inl.h +++ b/src/util-inl.h @@ -495,6 +495,13 @@ ArrayBufferViewContents::ArrayBufferViewContents( Read(value.As()); } +template +ArrayBufferViewContents::ArrayBufferViewContents( + v8::Local value) { + CHECK(value->IsArrayBufferView()); + Read(value.As()); +} + template ArrayBufferViewContents::ArrayBufferViewContents( v8::Local abv) { diff --git a/src/util.cc b/src/util.cc index 51f11b45396fa3..26dbfe844995eb 100644 --- a/src/util.cc +++ b/src/util.cc @@ -22,6 +22,7 @@ #include "util.h" // NOLINT(build/include_inline) #include "util-inl.h" +#include "env-inl.h" #include "node_buffer.h" #include "node_errors.h" #include "node_internals.h" diff --git a/src/util.h b/src/util.h index 5f02ffd2a3653e..a94e88f232fd31 100644 --- a/src/util.h +++ b/src/util.h @@ -181,8 +181,8 @@ void DumpBacktrace(FILE* fp); #endif -#define UNREACHABLE(expr) \ - ERROR_AND_ABORT("Unreachable code reached: " expr) +#define UNREACHABLE(...) \ + ERROR_AND_ABORT("Unreachable code reached" __VA_OPT__(": ") __VA_ARGS__) // TAILQ-style intrusive list node. template @@ -446,6 +446,7 @@ class ArrayBufferViewContents { ArrayBufferViewContents() = default; explicit inline ArrayBufferViewContents(v8::Local value); + explicit inline ArrayBufferViewContents(v8::Local value); explicit inline ArrayBufferViewContents(v8::Local abv); inline void Read(v8::Local abv); diff --git a/test/cctest/node_test_fixture.h b/test/cctest/node_test_fixture.h index f5740e5ce9278a..a396f14d18c6e4 100644 --- a/test/cctest/node_test_fixture.h +++ b/test/cctest/node_test_fixture.h @@ -7,7 +7,7 @@ #include "node.h" #include "node_platform.h" #include "node_internals.h" -#include "env.h" +#include "env-inl.h" #include "util-inl.h" #include "v8.h" #include "libplatform/libplatform.h" diff --git a/test/cctest/test_aliased_buffer.cc b/test/cctest/test_aliased_buffer.cc index 5421dd6d14582e..ba947700c1bf27 100644 --- a/test/cctest/test_aliased_buffer.cc +++ b/test/cctest/test_aliased_buffer.cc @@ -1,4 +1,3 @@ - #include "v8.h" #include "aliased_buffer.h" #include "node_test_fixture.h" diff --git a/test/cctest/test_per_process.cc b/test/cctest/test_per_process.cc new file mode 100644 index 00000000000000..43af8dd65a72d0 --- /dev/null +++ b/test/cctest/test_per_process.cc @@ -0,0 +1,34 @@ +#include "node_native_module.h" + +#include "gtest/gtest.h" +#include "node_test_fixture.h" + +#include + + +using node::native_module::NativeModuleLoader; +using node::native_module::NativeModuleRecordMap; + +class PerProcessTest : public ::testing::Test { + protected: + static const NativeModuleRecordMap get_sources_for_test() { + return NativeModuleLoader::instance_.source_; + } +}; + +namespace { + +TEST_F(PerProcessTest, EmbeddedSources) { + const auto& sources = PerProcessTest::get_sources_for_test(); + ASSERT_TRUE( + std::any_of(sources.cbegin(), sources.cend(), + [](auto p){ return p.second.is_one_byte(); })) + << "NativeModuleLoader::source_ should have some 8bit items"; + + ASSERT_TRUE( + std::any_of(sources.cbegin(), sources.cend(), + [](auto p){ return !p.second.is_one_byte(); })) + << "NativeModuleLoader::source_ should have some 16bit items"; +} + +} // end namespace diff --git a/test/cctest/test_platform.cc b/test/cctest/test_platform.cc index 876547480b7032..5420502124d6da 100644 --- a/test/cctest/test_platform.cc +++ b/test/cctest/test_platform.cc @@ -23,8 +23,10 @@ class RepostingTask : public v8::Task { ++*run_count_; if (repost_count_ > 0) { --repost_count_; - platform_->CallOnForegroundThread(isolate_, - new RepostingTask(repost_count_, run_count_, isolate_, platform_)); + std::shared_ptr task_runner = + platform_->GetForegroundTaskRunner(isolate_); + task_runner->PostTask(std::make_unique( + repost_count_, run_count_, isolate_, platform_)); } } @@ -43,8 +45,10 @@ TEST_F(PlatformTest, SkipNewTasksInFlushForegroundTasks) { const Argv argv; Env env {handle_scope, argv}; int run_count = 0; - platform->CallOnForegroundThread( - isolate_, new RepostingTask(2, &run_count, isolate_, platform.get())); + std::shared_ptr task_runner = + platform->GetForegroundTaskRunner(isolate_); + task_runner->PostTask( + std::make_unique(2, &run_count, isolate_, platform.get())); EXPECT_TRUE(platform->FlushForegroundTasks(isolate_)); EXPECT_EQ(1, run_count); EXPECT_TRUE(platform->FlushForegroundTasks(isolate_)); diff --git a/test/cctest/test_url.cc b/test/cctest/test_url.cc index ddef534b57485f..96f9741386360f 100644 --- a/test/cctest/test_url.cc +++ b/test/cctest/test_url.cc @@ -1,5 +1,6 @@ #include "node_url.h" #include "node_i18n.h" +#include "util-inl.h" #include "gtest/gtest.h" diff --git a/test/common/wpt.js b/test/common/wpt.js index 60be82564bf229..c227f2e6fa3529 100644 --- a/test/common/wpt.js +++ b/test/common/wpt.js @@ -41,16 +41,25 @@ class ResourceLoader { this.path = path; } - fetch(url, asPromise = true) { + /** + * Load a resource in test/fixtures/wpt specified with a URL + * @param {string} from the path of the file loading this resource, + * relative to thw WPT folder. + * @param {string} url the url of the resource being loaded. + * @param {boolean} asPromise if true, return the resource in a + * pseudo-Response object. + */ + read(from, url, asFetch = true) { // We need to patch this to load the WebIDL parser url = url.replace( '/resources/WebIDLParser.js', '/resources/webidl2/lib/webidl2.js' ); + const base = path.dirname(from); const file = url.startsWith('/') ? fixtures.path('wpt', url) : - fixtures.path('wpt', this.path, url); - if (asPromise) { + fixtures.path('wpt', base, url); + if (asFetch) { return fsPromises.readFile(file) .then((data) => { return { @@ -85,7 +94,7 @@ class StatusRule { * @returns {RegExp} */ transformPattern(pattern) { - const result = pattern.replace(/[-/\\^$+?.()|[\]{}]/g, '\\$&'); + const result = path.normalize(pattern).replace(/[-/\\^$+?.()|[\]{}]/g, '\\$&'); return new RegExp(result.replace('*', '.*')); } } @@ -155,8 +164,12 @@ class WPTTest { } } + getRelativePath() { + return path.join(this.module, this.filename); + } + getAbsolutePath() { - return fixtures.path('wpt', this.module, this.filename); + return fixtures.path('wpt', this.getRelativePath()); } getContent() { @@ -217,20 +230,41 @@ class StatusLoader { this.tests = []; } + /** + * Grep for all .*.js file recursively in a directory. + * @param {string} dir + */ + grep(dir) { + let result = []; + const list = fs.readdirSync(dir); + for (const file of list) { + const filepath = path.join(dir, file); + const stat = fs.statSync(filepath); + if (stat.isDirectory()) { + const list = this.grep(filepath); + result = result.concat(list); + } else { + if (!(/\.\w+\.js$/.test(filepath))) { + continue; + } + result.push(filepath); + } + } + return result; + } + load() { const dir = path.join(__dirname, '..', 'wpt'); const statusFile = path.join(dir, 'status', `${this.path}.json`); const result = JSON.parse(fs.readFileSync(statusFile, 'utf8')); this.rules.addRules(result); - const list = fs.readdirSync(fixtures.path('wpt', this.path)); - + const subDir = fixtures.path('wpt', this.path); + const list = this.grep(subDir); for (const file of list) { - if (!(/\.\w+\.js$/.test(file))) { - continue; - } - const match = this.rules.match(file); - this.tests.push(new WPTTest(this.path, file, match)); + const relativePath = path.relative(subDir, file); + const match = this.rules.match(relativePath); + this.tests.push(new WPTTest(this.path, relativePath, match)); } this.loaded = true; } @@ -309,8 +343,9 @@ class WPTRunner { const meta = test.title = this.getMeta(content); const absolutePath = test.getAbsolutePath(); - const context = this.generateContext(test.filename); - const code = this.mergeScripts(meta, content); + const context = this.generateContext(test); + const relativePath = test.getRelativePath(); + const code = this.mergeScripts(relativePath, meta, content); try { vm.runInContext(code, context, { filename: absolutePath @@ -327,16 +362,15 @@ class WPTRunner { this.tryFinish(); } - mock() { + mock(testfile) { const resource = this.resource; const result = { // This is a mock, because at the moment fetch is not implemented // in Node.js, but some tests and harness depend on this to pull // resources. fetch(file) { - return resource.fetch(file); + return resource.read(testfile, file, true); }, - location: {}, GLOBAL: { isWindow() { return false; } }, @@ -347,16 +381,17 @@ class WPTRunner { } // Note: this is how our global space for the WPT test should look like - getSandbox() { - const result = this.mock(); + getSandbox(filename) { + const result = this.mock(filename); for (const [name, desc] of this.globals) { Object.defineProperty(result, name, desc); } return result; } - generateContext(filename) { - const sandbox = this.sandbox = this.getSandbox(); + generateContext(test) { + const filename = test.filename; + const sandbox = this.sandbox = this.getSandbox(test.getRelativePath()); const context = this.context = vm.createContext(sandbox); const harnessPath = fixtures.path('wpt', 'resources', 'testharness.js'); @@ -375,8 +410,6 @@ class WPTRunner { // TODO(joyeecheung): we are not a window - work with the upstream to // add a new scope for us. - const { Worker } = require('worker_threads'); - sandbox.DedicatedWorker = Worker; // Pretend we are a Worker return context; } @@ -512,7 +545,7 @@ class WPTRunner { } } - mergeScripts(meta, content) { + mergeScripts(base, meta, content) { if (!meta.script) { return content; } @@ -520,7 +553,7 @@ class WPTRunner { // only one script let result = ''; for (const script of meta.script) { - result += this.resource.fetch(script, false); + result += this.resource.read(base, script, false); } return result + content; diff --git a/test/es-module/test-esm-json-cache.mjs b/test/es-module/test-esm-json-cache.mjs index d1fee4f444c2c0..0f7d6804061bcd 100644 --- a/test/es-module/test-esm-json-cache.mjs +++ b/test/es-module/test-esm-json-cache.mjs @@ -1,4 +1,4 @@ -// Flags: --experimental-modules --experimental-json-modules +// Flags: --experimental-modules import '../common/index.mjs'; import { strictEqual, deepStrictEqual } from 'assert'; diff --git a/test/es-module/test-esm-json.mjs b/test/es-module/test-esm-json.mjs index 3d246124a9bdae..f196945b4b68f4 100644 --- a/test/es-module/test-esm-json.mjs +++ b/test/es-module/test-esm-json.mjs @@ -1,4 +1,5 @@ -// Flags: --experimental-modules --experimental-json-modules +// Flags: --experimental-modules + import '../common/index.mjs'; import { strictEqual } from 'assert'; diff --git a/test/fixtures/GH-892-request.js b/test/fixtures/GH-892-request.js index 19e1eea96c3d84..5c333394376b40 100644 --- a/test/fixtures/GH-892-request.js +++ b/test/fixtures/GH-892-request.js @@ -37,7 +37,7 @@ var options = { }; var req = https.request(options, function(res) { - assert.strictEqual(200, res.statusCode); + assert.strictEqual(res.statusCode, 200); gotResponse = true; console.error('DONE'); res.resume(); diff --git a/test/fixtures/workload/allocation-exit.js b/test/fixtures/workload/allocation-exit.js new file mode 100644 index 00000000000000..dccc61ac94f9fa --- /dev/null +++ b/test/fixtures/workload/allocation-exit.js @@ -0,0 +1,17 @@ +'use strict'; + +const util = require('util'); +const total = parseInt(process.env.TEST_ALLOCATION) || 100; +let count = 0; +let string = ''; +function runAllocation() { + string += util.inspect(process.env); + if (count++ < total) { + setTimeout(runAllocation, 1); + } else { + console.log(string.length); + process.exit(55); + } +} + +setTimeout(runAllocation, 1); diff --git a/test/fixtures/workload/allocation-sigint.js b/test/fixtures/workload/allocation-sigint.js new file mode 100644 index 00000000000000..96ae669016a91b --- /dev/null +++ b/test/fixtures/workload/allocation-sigint.js @@ -0,0 +1,17 @@ +'use strict'; + +const util = require('util'); +const total = parseInt(process.env.TEST_ALLOCATION) || 100; +let count = 0; +let string = ''; +function runAllocation() { + string += util.inspect(process.env); + if (count++ < total) { + setTimeout(runAllocation, 1); + } else { + console.log(string.length); + process.kill(process.pid, "SIGINT"); + } +} + +setTimeout(runAllocation, 1); diff --git a/test/fixtures/workload/allocation-worker-argv.js b/test/fixtures/workload/allocation-worker-argv.js new file mode 100644 index 00000000000000..299eb884e51442 --- /dev/null +++ b/test/fixtures/workload/allocation-worker-argv.js @@ -0,0 +1,11 @@ +'use strict'; + +const { Worker } = require('worker_threads'); +const path = require('path'); +new Worker(path.join(__dirname, 'allocation.js'), { + execArgv: [ + '--heap-prof', + '--heap-prof-interval', + process.HEAP_PROF_INTERVAL || '128', + ] +}); diff --git a/test/fixtures/workload/allocation-worker.js b/test/fixtures/workload/allocation-worker.js new file mode 100644 index 00000000000000..21be6ce91a35a9 --- /dev/null +++ b/test/fixtures/workload/allocation-worker.js @@ -0,0 +1,5 @@ +'use strict'; + +const { Worker } = require('worker_threads'); +const path = require('path'); +new Worker(path.join(__dirname, 'allocation.js')); diff --git a/test/fixtures/workload/allocation.js b/test/fixtures/workload/allocation.js new file mode 100644 index 00000000000000..b9a767f0f5b10e --- /dev/null +++ b/test/fixtures/workload/allocation.js @@ -0,0 +1,16 @@ +'use strict'; + +const util = require('util'); +const total = parseInt(process.env.TEST_ALLOCATION) || 100; +let count = 0; +let string = ''; +function runAllocation() { + string += util.inspect(process.env); + if (count++ < total) { + setTimeout(runAllocation, 1); + } else { + console.log(string.length); + } +} + +setTimeout(runAllocation, 1); diff --git a/test/fixtures/wpt/README.md b/test/fixtures/wpt/README.md index 51450f918bd1b8..802e20d816711c 100644 --- a/test/fixtures/wpt/README.md +++ b/test/fixtures/wpt/README.md @@ -11,9 +11,9 @@ See [test/wpt](../../wpt/README.md) for information on how these tests are run. Last update: - console: https://github.com/web-platform-tests/wpt/tree/9786a4b131/console -- encoding: https://github.com/web-platform-tests/wpt/tree/a093a659ed/encoding -- url: https://github.com/web-platform-tests/wpt/tree/75b0f336c5/url -- resources: https://github.com/web-platform-tests/wpt/tree/679a364421/resources +- encoding: https://github.com/web-platform-tests/wpt/tree/7287608f90/encoding +- url: https://github.com/web-platform-tests/wpt/tree/418f7fabeb/url +- resources: https://github.com/web-platform-tests/wpt/tree/e1fddfbf80/resources - interfaces: https://github.com/web-platform-tests/wpt/tree/712c9f275e/interfaces - html/webappapis/microtask-queuing: https://github.com/web-platform-tests/wpt/tree/0c3bed38df/html/webappapis/microtask-queuing - html/webappapis/timers: https://github.com/web-platform-tests/wpt/tree/ddfe9c089b/html/webappapis/timers diff --git a/test/fixtures/wpt/encoding/encodeInto.any.js b/test/fixtures/wpt/encoding/encodeInto.any.js new file mode 100644 index 00000000000000..fda0d1b72ce787 --- /dev/null +++ b/test/fixtures/wpt/encoding/encodeInto.any.js @@ -0,0 +1,149 @@ +[ + { + "input": "Hi", + "read": 0, + "destinationLength": 0, + "written": [] + }, + { + "input": "A", + "read": 1, + "destinationLength": 10, + "written": [0x41] + }, + { + "input": "\u{1D306}", // "\uD834\uDF06" + "read": 2, + "destinationLength": 4, + "written": [0xF0, 0x9D, 0x8C, 0x86] + }, + { + "input": "\u{1D306}A", + "read": 0, + "destinationLength": 3, + "written": [] + }, + { + "input": "\uD834A\uDF06A¥Hi", + "read": 5, + "destinationLength": 10, + "written": [0xEF, 0xBF, 0xBD, 0x41, 0xEF, 0xBF, 0xBD, 0x41, 0xC2, 0xA5] + }, + { + "input": "A\uDF06", + "read": 2, + "destinationLength": 4, + "written": [0x41, 0xEF, 0xBF, 0xBD] + }, + { + "input": "¥¥", + "read": 2, + "destinationLength": 4, + "written": [0xC2, 0xA5, 0xC2, 0xA5] + } +].forEach(testData => { + [ + { + "bufferIncrease": 0, + "destinationOffset": 0, + "filler": 0 + }, + { + "bufferIncrease": 10, + "destinationOffset": 4, + "filler": 0 + }, + { + "bufferIncrease": 0, + "destinationOffset": 0, + "filler": 0x80 + }, + { + "bufferIncrease": 10, + "destinationOffset": 4, + "filler": 0x80 + }, + { + "bufferIncrease": 0, + "destinationOffset": 0, + "filler": "random" + }, + { + "bufferIncrease": 10, + "destinationOffset": 4, + "filler": "random" + } + ].forEach(destinationData => { + test(() => { + // Setup + const bufferLength = testData.destinationLength + destinationData.bufferIncrease, + destinationOffset = destinationData.destinationOffset, + destinationLength = testData.destinationLength, + destinationFiller = destinationData.filler, + encoder = new TextEncoder(), + buffer = new ArrayBuffer(bufferLength), + view = new Uint8Array(buffer, destinationOffset, destinationLength), + fullView = new Uint8Array(buffer), + control = new Array(bufferLength); + let byte = destinationFiller; + for (let i = 0; i < bufferLength; i++) { + if (destinationFiller === "random") { + byte = Math.floor(Math.random() * 256); + } + control[i] = byte; + fullView[i] = byte; + } + + // It's happening + const result = encoder.encodeInto(testData.input, view); + + // Basics + assert_equals(view.byteLength, destinationLength); + assert_equals(view.length, destinationLength); + + // Remainder + assert_equals(result.read, testData.read); + assert_equals(result.written, testData.written.length); + for (let i = 0; i < bufferLength; i++) { + if (i < destinationOffset || i >= (destinationOffset + testData.written.length)) { + assert_equals(fullView[i], control[i]); + } else { + assert_equals(fullView[i], testData.written[i - destinationOffset]); + } + } + }, "encodeInto() with " + testData.input + " and destination length " + testData.destinationLength + ", offset " + destinationData.destinationOffset + ", filler " + destinationData.filler); + }); +}); + +[DataView, + Int8Array, + Int16Array, + Int32Array, + Uint16Array, + Uint32Array, + Uint8ClampedArray, + Float32Array, + Float64Array].forEach(view => { + test(() => { + assert_throws(new TypeError(), () => new TextEncoder().encodeInto("", new view(new ArrayBuffer(0)))); + }, "Invalid encodeInto() destination: " + view); + }); + +test(() => { + assert_throws(new TypeError(), () => new TextEncoder().encodeInto("", new ArrayBuffer(10))); +}, "Invalid encodeInto() destination: ArrayBuffer"); + +test(() => { + const buffer = new ArrayBuffer(10), + view = new Uint8Array(buffer); + let { read, written } = new TextEncoder().encodeInto("", view); + assert_equals(read, 0); + assert_equals(written, 0); + new MessageChannel().port1.postMessage(buffer, [buffer]); + ({ read, written } = new TextEncoder().encodeInto("", view)); + assert_equals(read, 0); + assert_equals(written, 0); + ({ read, written } = new TextEncoder().encodeInto("test", view)); + assert_equals(read, 0); + assert_equals(written, 0); +}, "encodeInto() and a detached output buffer"); diff --git a/test/fixtures/wpt/encoding/streams/decode-bad-chunks.any.js b/test/fixtures/wpt/encoding/streams/decode-bad-chunks.any.js index 101fb3aeb614cf..de2ba74c1016a1 100644 --- a/test/fixtures/wpt/encoding/streams/decode-bad-chunks.any.js +++ b/test/fixtures/wpt/encoding/streams/decode-bad-chunks.any.js @@ -23,26 +23,6 @@ const badChunks = [ name: 'array', value: [65] }, - { - name: 'detached ArrayBufferView', - value: (() => { - const u8 = new Uint8Array([65]); - const ab = u8.buffer; - const mc = new MessageChannel(); - mc.port1.postMessage(ab, [ab]); - return u8; - })() - }, - { - name: 'detached ArrayBuffer', - value: (() => { - const u8 = new Uint8Array([65]); - const ab = u8.buffer; - const mc = new MessageChannel(); - mc.port1.postMessage(ab, [ab]); - return ab; - })() - }, { name: 'SharedArrayBuffer', // Use a getter to postpone construction so that all tests don't fail where diff --git a/test/fixtures/wpt/encoding/streams/decode-utf8.any.js b/test/fixtures/wpt/encoding/streams/decode-utf8.any.js index 34fa764bf0a682..266899c6c965f7 100644 --- a/test/fixtures/wpt/encoding/streams/decode-utf8.any.js +++ b/test/fixtures/wpt/encoding/streams/decode-utf8.any.js @@ -39,3 +39,23 @@ promise_test(async () => { assert_array_equals(array, [expectedOutputString], 'the output should be in one chunk'); }, 'a trailing empty chunk should be ignored'); + +promise_test(async () => { + const buffer = new ArrayBuffer(3); + const view = new Uint8Array(buffer, 1, 1); + view[0] = 65; + new MessageChannel().port1.postMessage(buffer, [buffer]); + const input = readableStreamFromArray([view]); + const output = input.pipeThrough(new TextDecoderStream()); + const array = await readableStreamToArray(output); + assert_array_equals(array, [], 'no chunks should be output'); +}, 'decoding a transferred Uint8Array chunk should give no output'); + +promise_test(async () => { + const buffer = new ArrayBuffer(1); + new MessageChannel().port1.postMessage(buffer, [buffer]); + const input = readableStreamFromArray([buffer]); + const output = input.pipeThrough(new TextDecoderStream()); + const array = await readableStreamToArray(output); + assert_array_equals(array, [], 'no chunks should be output'); +}, 'decoding a transferred ArrayBuffer chunk should give no output'); diff --git a/test/fixtures/wpt/encoding/textdecoder-fatal-single-byte.any.js b/test/fixtures/wpt/encoding/textdecoder-fatal-single-byte.any.js index 9d12134edc58d8..d3e9ae9c9a7774 100644 --- a/test/fixtures/wpt/encoding/textdecoder-fatal-single-byte.any.js +++ b/test/fixtures/wpt/encoding/textdecoder-fatal-single-byte.any.js @@ -1,4 +1,6 @@ +// META: timeout=long // META: title=Encoding API: Fatal flag for single byte encodings +// META: timeout=long var singleByteEncodings = [ {encoding: 'IBM866', bad: []}, diff --git a/test/fixtures/wpt/resources/idlharness.js b/test/fixtures/wpt/resources/idlharness.js index f1a39ae9f2692f..c7a040996b9994 100644 --- a/test/fixtures/wpt/resources/idlharness.js +++ b/test/fixtures/wpt/resources/idlharness.js @@ -298,7 +298,9 @@ IdlArray.prototype.add_dependency_idls = function(raw_idls, options) }.bind(this)); deps.forEach(function(name) { - new_options.only.push(name); + if (!new_options.only.includes(name)) { + new_options.only.push(name); + } const follow_up = new Set(); for (const dep_type of ["inheritance", "implements", "includes"]) { @@ -306,13 +308,19 @@ IdlArray.prototype.add_dependency_idls = function(raw_idls, options) const inheriting = parsed[dep_type]; const inheritor = parsed.name || parsed.target; const deps = [inheriting]; - // For A includes B, we can ignore A unless B is being tested. + // For A includes B, we can ignore A, unless B (or some of its + // members) is being tested. if (dep_type !== "includes" - || (inheriting in this.members && !this.members[inheriting].untested)) { + || inheriting in this.members && !this.members[inheriting].untested + || this.partials.some(function(p) { + return p.name === inheriting; + })) { deps.push(inheritor); } for (const dep of deps) { - new_options.only.push(dep); + if (!new_options.only.includes(dep)) { + new_options.only.push(dep); + } all_deps.add(dep); follow_up.add(dep); } @@ -718,9 +726,6 @@ function exposed_in(globals) { self instanceof ServiceWorkerGlobalScope) { return globals.has("ServiceWorker"); } - if ('NodeScope' in self) { - return true; - } throw new IdlHarnessError("Unexpected global object"); } @@ -1669,7 +1674,122 @@ IdlInterface.prototype.test_self = function() }.bind(this), this.name + " interface: legacy window alias"); } - // TODO: Test named constructors if I find any interfaces that have them. + + if (this.has_extended_attribute("NamedConstructor")) { + var constructors = this.extAttrs + .filter(function(attr) { return attr.name == "NamedConstructor"; }); + if (constructors.length !== 1) { + throw new IdlHarnessError("Internal error: missing support for multiple NamedConstructor extended attributes"); + } + var constructor = constructors[0]; + var min_length = minOverloadLength([constructor]); + + subsetTestByKey(this.name, test, function() + { + // This function tests WebIDL as of 2019-01-14. + + // "for every [NamedConstructor] extended attribute on an exposed + // interface, a corresponding property must exist on the ECMAScript + // global object. The name of the property is the + // [NamedConstructor]'s identifier, and its value is an object + // called a named constructor, ... . The property has the attributes + // { [[Writable]]: true, [[Enumerable]]: false, + // [[Configurable]]: true }." + var name = constructor.rhs.value; + assert_own_property(self, name); + var desc = Object.getOwnPropertyDescriptor(self, name); + assert_equals(desc.value, self[name], "wrong value in " + name + " property descriptor"); + assert_true(desc.writable, name + " should be writable"); + assert_false(desc.enumerable, name + " should not be enumerable"); + assert_true(desc.configurable, name + " should be configurable"); + assert_false("get" in desc, name + " should not have a getter"); + assert_false("set" in desc, name + " should not have a setter"); + }.bind(this), this.name + " interface: named constructor"); + + subsetTestByKey(this.name, test, function() + { + // This function tests WebIDL as of 2019-01-14. + + // "2. Let F be ! CreateBuiltinFunction(realm, steps, + // realm.[[Intrinsics]].[[%FunctionPrototype%]])." + var name = constructor.rhs.value; + var value = self[name]; + assert_equals(typeof value, "function", "type of value in " + name + " property descriptor"); + assert_not_equals(value, this.get_interface_object(), "wrong value in " + name + " property descriptor"); + assert_equals(Object.getPrototypeOf(value), Function.prototype, "wrong value for " + name + "'s prototype"); + }.bind(this), this.name + " interface: named constructor object"); + + subsetTestByKey(this.name, test, function() + { + // This function tests WebIDL as of 2019-01-14. + + // "7. Let proto be the interface prototype object of interface I + // in realm. + // "8. Perform ! DefinePropertyOrThrow(F, "prototype", + // PropertyDescriptor{ + // [[Value]]: proto, [[Writable]]: false, + // [[Enumerable]]: false, [[Configurable]]: false + // })." + var name = constructor.rhs.value; + var expected = this.get_interface_object().prototype; + var desc = Object.getOwnPropertyDescriptor(self[name], "prototype"); + assert_equals(desc.value, expected, "wrong value for " + name + ".prototype"); + assert_false(desc.writable, "prototype should not be writable"); + assert_false(desc.enumerable, "prototype should not be enumerable"); + assert_false(desc.configurable, "prototype should not be configurable"); + assert_false("get" in desc, "prototype should not have a getter"); + assert_false("set" in desc, "prototype should not have a setter"); + }.bind(this), this.name + " interface: named constructor prototype property"); + + subsetTestByKey(this.name, test, function() + { + // This function tests WebIDL as of 2019-01-14. + + // "3. Perform ! SetFunctionName(F, id)." + var name = constructor.rhs.value; + var desc = Object.getOwnPropertyDescriptor(self[name], "name"); + assert_equals(desc.value, name, "wrong value for " + name + ".name"); + assert_false(desc.writable, "name should not be writable"); + assert_false(desc.enumerable, "name should not be enumerable"); + assert_true(desc.configurable, "name should be configurable"); + assert_false("get" in desc, "name should not have a getter"); + assert_false("set" in desc, "name should not have a setter"); + }.bind(this), this.name + " interface: named constructor name"); + + subsetTestByKey(this.name, test, function() + { + // This function tests WebIDL as of 2019-01-14. + + // "4. Initialize S to the effective overload set for constructors + // with identifier id on interface I and with argument count 0. + // "5. Let length be the length of the shortest argument list of + // the entries in S. + // "6. Perform ! SetFunctionLength(F, length)." + var name = constructor.rhs.value; + var desc = Object.getOwnPropertyDescriptor(self[name], "length"); + assert_equals(desc.value, min_length, "wrong value for " + name + ".length"); + assert_false(desc.writable, "length should not be writable"); + assert_false(desc.enumerable, "length should not be enumerable"); + assert_true(desc.configurable, "length should be configurable"); + assert_false("get" in desc, "length should not have a getter"); + assert_false("set" in desc, "length should not have a setter"); + }.bind(this), this.name + " interface: named constructor length"); + + subsetTestByKey(this.name, test, function() + { + // This function tests WebIDL as of 2019-01-14. + + // "1. Let steps be the following steps: + // " 1. If NewTarget is undefined, then throw a TypeError." + var name = constructor.rhs.value; + var args = constructor.arguments.map(function(arg) { + return create_suitable_object(arg.idlType); + }); + assert_throws(new TypeError(), function() { + self[name](...args); + }.bind(this)); + }.bind(this), this.name + " interface: named constructor without 'new'"); + } subsetTestByKey(this.name, test, function() { @@ -2304,22 +2424,7 @@ IdlInterface.prototype.do_member_operation_asserts = function(memberHolderObject } } -IdlInterface.prototype.add_iterable_members = function(member) -{ - this.members.push(new IdlInterfaceMember( - { type: "operation", name: "entries", idlType: "iterator", arguments: []})); - this.members.push(new IdlInterfaceMember( - { type: "operation", name: "keys", idlType: "iterator", arguments: []})); - this.members.push(new IdlInterfaceMember( - { type: "operation", name: "values", idlType: "iterator", arguments: []})); - this.members.push(new IdlInterfaceMember( - { type: "operation", name: "forEach", idlType: "void", - arguments: - [{ name: "callback", idlType: {idlType: "function"}}, - { name: "thisValue", idlType: {idlType: "any"}, optional: true}]})); -}; - -IdlInterface.prototype.test_to_json_operation = function(memberHolderObject, member) { +IdlInterface.prototype.test_to_json_operation = function(desc, memberHolderObject, member) { var instanceName = memberHolderObject && memberHolderObject.constructor.name || member.name + " object"; if (member.has_extended_attribute("Default")) { @@ -2335,38 +2440,40 @@ IdlInterface.prototype.test_to_json_operation = function(memberHolderObject, mem this.array.assert_type_is(json[k], type); delete json[k]; }, this); - }.bind(this), "Test default toJSON operation of " + instanceName); + }.bind(this), this.name + " interface: default toJSON operation on " + desc); } else { subsetTestByKey(this.name, test, function() { assert_true(this.array.is_json_type(member.idlType), JSON.stringify(member.idlType) + " is not an appropriate return value for the toJSON operation of " + instanceName); this.array.assert_type_is(memberHolderObject.toJSON(), member.idlType); - }.bind(this), "Test toJSON operation of " + instanceName); + }.bind(this), this.name + " interface: toJSON operation on " + desc); } }; IdlInterface.prototype.test_member_iterable = function(member) { - var isPairIterator = member.idlType.length === 2; subsetTestByKey(this.name, test, function() { - var descriptor = Object.getOwnPropertyDescriptor(this.get_interface_object().prototype, Symbol.iterator); - assert_true(descriptor.writable, "property should be writable"); - assert_true(descriptor.configurable, "property should be configurable"); - assert_false(descriptor.enumerable, "property should not be enumerable"); - assert_equals(this.get_interface_object().prototype[Symbol.iterator].name, isPairIterator ? "entries" : "values", "@@iterator function does not have the right name"); - }.bind(this), "Testing Symbol.iterator property of iterable interface " + this.name); - - if (isPairIterator) { - subsetTestByKey(this.name, test, function() { - assert_equals(this.get_interface_object().prototype[Symbol.iterator], this.get_interface_object().prototype["entries"], "entries method is not the same as @@iterator"); - }.bind(this), "Testing pair iterable interface " + this.name); - } else { - subsetTestByKey(this.name, test, function() { - ["entries", "keys", "values", "forEach", Symbol.Iterator].forEach(function(property) { - assert_equals(this.get_interface_object().prototype[property], Array.prototype[property], property + " function is not the same as Array one"); + var isPairIterator = member.idlType.length === 2; + var proto = this.get_interface_object().prototype; + var descriptor = Object.getOwnPropertyDescriptor(proto, Symbol.iterator); + + assert_true(descriptor.writable, "@@iterator property should be writable"); + assert_true(descriptor.configurable, "@@iterator property should be configurable"); + assert_false(descriptor.enumerable, "@@iterator property should not be enumerable"); + assert_equals(typeof descriptor.value, "function", "@@iterator property should be a function"); + assert_equals(descriptor.value.length, 0, "@@iterator function object length should be 0"); + assert_equals(descriptor.value.name, isPairIterator ? "entries" : "values", "@@iterator function object should have the right name"); + + if (isPairIterator) { + assert_equals(proto["entries"], proto[Symbol.iterator], "entries method should be the same as @@iterator method"); + } else { + assert_equals(proto[Symbol.iterator], Array.prototype[Symbol.iterator], "@@iterator method should be the same as Array prototype's"); + ["entries", "keys", "values", "forEach", Symbol.iterator].forEach(function(property) { + var propertyName = property === Symbol.iterator ? "@@iterator" : property; + assert_equals(proto[property], Array.prototype[property], propertyName + " method should be the same as Array prototype's"); }.bind(this)); - }.bind(this), "Testing value iterable interface " + this.name); - } + } + }.bind(this), this.name + " interface: iterable<" + member.idlType.map(function(t) { return t.idlType; }).join(", ") + ">"); }; IdlInterface.prototype.test_member_stringifier = function(member) @@ -2432,19 +2539,6 @@ IdlInterface.prototype.test_member_stringifier = function(member) IdlInterface.prototype.test_members = function() { - for (var i = 0; i < this.members.length; i++) - { - var member = this.members[i]; - switch (member.type) { - case "iterable": - this.add_iterable_members(member); - break; - // TODO: add setlike and maplike handling. - default: - break; - } - } - for (var i = 0; i < this.members.length; i++) { var member = this.members[i]; @@ -2743,7 +2837,7 @@ IdlInterface.prototype.test_interface_of = function(desc, obj, exception, expect } if (member.is_to_json_regular_operation()) { - this.test_to_json_operation(obj, member); + this.test_to_json_operation(desc, obj, member); } } }; diff --git a/test/fixtures/wpt/resources/testdriver-actions.js b/test/fixtures/wpt/resources/testdriver-actions.js index 46c68858e45746..43d8b1df00ae4c 100644 --- a/test/fixtures/wpt/resources/testdriver-actions.js +++ b/test/fixtures/wpt/resources/testdriver-actions.js @@ -7,7 +7,7 @@ function Actions() { this.sourceTypes = new Map([["key", KeySource], ["pointer", PointerSource], - ["general", GeneralSource]]); + ["none", GeneralSource]]); this.sources = new Map(); this.sourceOrder = []; for (let sourceType of this.sourceTypes.keys()) { @@ -17,11 +17,19 @@ for (let sourceType of this.sourceTypes.keys()) { this.currentSources.set(sourceType, null); } - this.createSource("general"); + this.createSource("none"); this.tickIdx = 0; } Actions.prototype = { + ButtonType: { + LEFT: 0, + MIDDLE: 1, + RIGHT: 2, + BACK: 3, + FORWARD: 4, + }, + /** * Generate the action sequence suitable for passing to * test_driver.action_sequence @@ -62,7 +70,7 @@ * If no name is passed, a new source with the given type is * created. * - * @param {String} type - Source type ('general', 'key', or 'pointer') + * @param {String} type - Source type ('none', 'key', or 'pointer') * @param {String?} name - Name of the source * @returns {Source} Source object for that source. */ @@ -98,7 +106,7 @@ * @returns {Actions} */ addKeyboard: function(name, set=true) { - this.createSource("key", name, true); + this.createSource("key", name); if (set) { this.setKeyboard(name); } @@ -125,7 +133,7 @@ * @returns {Actions} */ addPointer: function(name, pointerType="mouse", set=true) { - this.createSource("pointer", name, true, {pointerType: pointerType}); + this.createSource("pointer", name, {pointerType: pointerType}); if (set) { this.setPointer(name); } @@ -187,7 +195,7 @@ * @returns {Actions} */ pause: function(duration) { - this.getSource("general").addPause(this, duration); + this.getSource("none").addPause(this, duration); return this; }, @@ -225,7 +233,7 @@ * pointer source * @returns {Actions} */ - pointerDown: function({button=0, sourceName=null}={}) { + pointerDown: function({button=this.ButtonType.LEFT, sourceName=null}={}) { let source = this.getSource("pointer", sourceName); source.pointerDown(this, button); return this; @@ -239,7 +247,7 @@ * source * @returns {Actions} */ - pointerUp: function({button=0, sourceName=null}={}) { + pointerUp: function({button=this.ButtonType.LEFT, sourceName=null}={}) { let source = this.getSource("pointer", sourceName); source.pointerUp(this, button); return this; diff --git a/test/fixtures/wpt/resources/testdriver.js b/test/fixtures/wpt/resources/testdriver.js index e0741e8d61d4d6..031be1b7e55016 100644 --- a/test/fixtures/wpt/resources/testdriver.js +++ b/test/fixtures/wpt/resources/testdriver.js @@ -192,32 +192,89 @@ * @returns {Promise} fufiled after the actions are performed, or rejected in * the cases the WebDriver command errors */ - action_sequence(actions) { + action_sequence: function(actions) { return window.test_driver_internal.action_sequence(actions); + }, + + /** + * Generates a test report on the current page + * + * The generate_test_report function generates a report (to be observed + * by ReportingObserver) for testing purposes, as described in + * {@link https://w3c.github.io/reporting/#generate-test-report-command} + * + * @returns {Promise} fulfilled after the report is generated, or + * rejected if the report generation fails + */ + generate_test_report: function(message) { + return window.test_driver_internal.generate_test_report(message); } }; window.test_driver_internal = { /** - * Triggers a user-initiated click + * This flag should be set to `true` by any code which implements the + * internal methods defined below for automation purposes. Doing so + * allows the library to signal failure immediately when an automated + * implementation of one of the methods is not available. + */ + in_automation: false, + + /** + * Waits for a user-initiated click * * @param {Element} element - element to be clicked * @param {{x: number, y: number} coords - viewport coordinates to click at - * @returns {Promise} fulfilled after click occurs or rejected if click fails + * @returns {Promise} fulfilled after click occurs */ click: function(element, coords) { - return Promise.reject(new Error("unimplemented")); + if (this.in_automation) { + return Promise.reject(new Error('Not implemented')); + } + + return new Promise(function(resolve, reject) { + element.addEventListener("click", resolve); + }); }, /** - * Triggers a user-initiated click + * Waits for an element to receive a series of key presses * - * @param {Element} element - element to be clicked - * @param {String} keys - keys to send to the element - * @returns {Promise} fulfilled after keys are sent or rejected if click fails + * @param {Element} element - element which should receve key presses + * @param {String} keys - keys to expect + * @returns {Promise} fulfilled after keys are received or rejected if + * an incorrect key sequence is received */ send_keys: function(element, keys) { - return Promise.reject(new Error("unimplemented")); + if (this.in_automation) { + return Promise.reject(new Error('Not implemented')); + } + + return new Promise(function(resolve, reject) { + var seen = ""; + + function remove() { + element.removeEventListener("keydown", onKeyDown); + } + + function onKeyDown(event) { + if (event.key.length > 1) { + return; + } + + seen += event.key; + + if (keys.indexOf(seen) !== 0) { + reject(new Error("Unexpected key sequence: " + seen)); + remove(); + } else if (seen === keys) { + resolve(); + remove(); + } + } + + element.addEventListener("keydown", onKeyDown); + }); }, /** @@ -238,6 +295,17 @@ */ action_sequence: function(actions) { return Promise.reject(new Error("unimplemented")); + }, + + /** + * Generates a test report on the current page + * + * @param {String} message - the message to be contained in the report + * @returns {Promise} fulfilled after the report is generated, or + * rejected if the report generation fails + */ + generate_test_report: function(message) { + return Promise.reject(new Error("unimplemented")); } }; })(); diff --git a/test/fixtures/wpt/resources/testharness.js b/test/fixtures/wpt/resources/testharness.js index 18a6f70beab26b..bffdf022b33255 100644 --- a/test/fixtures/wpt/resources/testharness.js +++ b/test/fixtures/wpt/resources/testharness.js @@ -513,7 +513,7 @@ policies and contribution forms [3]. return new DedicatedWorkerTestEnvironment(); } - if (!('self' in global_scope)) { + if (!('location' in global_scope)) { return new ShellTestEnvironment(); } @@ -635,7 +635,7 @@ policies and contribution forms [3]. * which can make it a lot easier to test a very specific series of events, * including ensuring that unexpected events are not fired at any point. */ - function EventWatcher(test, watchedNode, eventTypes) + function EventWatcher(test, watchedNode, eventTypes, timeoutPromise) { if (typeof eventTypes == 'string') { eventTypes = [eventTypes]; @@ -712,6 +712,27 @@ policies and contribution forms [3]. recordedEvents = []; } return new Promise(function(resolve, reject) { + var timeout = test.step_func(function() { + // If the timeout fires after the events have been received + // or during a subsequent call to wait_for, ignore it. + if (!waitingFor || waitingFor.resolve !== resolve) + return; + + // This should always fail, otherwise we should have + // resolved the promise. + assert_true(waitingFor.types.length == 0, + 'Timed out waiting for ' + waitingFor.types.join(', ')); + var result = recordedEvents; + recordedEvents = null; + var resolveFunc = waitingFor.resolve; + waitingFor = null; + resolveFunc(result); + }); + + if (timeoutPromise) { + timeoutPromise().then(timeout); + } + waitingFor = { types: types, resolve: resolve, @@ -1111,7 +1132,7 @@ policies and contribution forms [3]. assert(Math.abs(actual[i] - expected[i]) <= epsilon, "assert_array_approx_equals", description, "property ${i}, expected ${expected} +/- ${epsilon}, expected ${expected} but got ${actual}", - {i:i, expected:expected[i], actual:actual[i]}); + {i:i, expected:expected[i], actual:actual[i], epsilon:epsilon}); } } expose(assert_array_approx_equals, "assert_array_approx_equals"); @@ -1478,7 +1499,7 @@ policies and contribution forms [3]. } this.name = name; - this.phase = tests.is_aborted ? + this.phase = (tests.is_aborted || tests.phase === tests.phases.COMPLETE) ? this.phases.COMPLETE : this.phases.INITIAL; this.status = this.NOTRUN; @@ -1486,11 +1507,9 @@ policies and contribution forms [3]. this.index = null; this.properties = properties; - var timeout = properties.timeout ? properties.timeout : settings.test_timeout; - if (timeout !== null) { - this.timeout_length = timeout * tests.timeout_multiplier; - } else { - this.timeout_length = null; + this.timeout_length = settings.test_timeout; + if (this.timeout_length !== null) { + this.timeout_length *= tests.timeout_multiplier; } this.message = null; @@ -1503,6 +1522,13 @@ policies and contribution forms [3]. this._user_defined_cleanup_count = 0; this._done_callbacks = []; + // Tests declared following harness completion are likely an indication + // of a programming error, but they cannot be reported + // deterministically. + if (tests.phase === tests.phases.COMPLETE) { + return; + } + tests.push(this); } @@ -1904,7 +1930,9 @@ policies and contribution forms [3]. */ function RemoteContext(remote, message_target, message_filter) { this.running = true; + this.started = false; this.tests = new Array(); + this.early_exception = null; var this_obj = this; // If remote context is cross origin assigning to onerror is not @@ -1943,6 +1971,21 @@ policies and contribution forms [3]. } RemoteContext.prototype.remote_error = function(error) { + if (error.preventDefault) { + error.preventDefault(); + } + + // Defer interpretation of errors until the testing protocol has + // started and the remote test's `allow_uncaught_exception` property + // is available. + if (!this.started) { + this.early_exception = error; + } else if (!this.allow_uncaught_exception) { + this.report_uncaught(error); + } + }; + + RemoteContext.prototype.report_uncaught = function(error) { var message = error.message || String(error); var filename = (error.filename ? " " + error.filename: ""); // FIXME: Display remote error states separately from main document @@ -1950,9 +1993,14 @@ policies and contribution forms [3]. tests.set_status(tests.status.ERROR, "Error in remote" + filename + ": " + message, error.stack); + }; - if (error.preventDefault) { - error.preventDefault(); + RemoteContext.prototype.start = function(data) { + this.started = true; + this.allow_uncaught_exception = data.properties.allow_uncaught_exception; + + if (this.early_exception && !this.allow_uncaught_exception) { + this.report_uncaught(this.early_exception); } }; @@ -2002,6 +2050,7 @@ policies and contribution forms [3]. }; RemoteContext.prototype.message_handlers = { + start: RemoteContext.prototype.start, test_state: RemoteContext.prototype.test_state, result: RemoteContext.prototype.test_done, complete: RemoteContext.prototype.remote_done @@ -2114,6 +2163,9 @@ policies and contribution forms [3]. } } else if (p == "timeout_multiplier") { this.timeout_multiplier = value; + if (this.timeout_length) { + this.timeout_length *= this.timeout_multiplier; + } } } } @@ -2338,6 +2390,42 @@ policies and contribution forms [3]. return duplicates; }; + function code_unit_str(char) { + return 'U+' + char.charCodeAt(0).toString(16); + } + + function sanitize_unpaired_surrogates(str) { + return str.replace(/([\ud800-\udbff])(?![\udc00-\udfff])/g, + function(_, unpaired) + { + return code_unit_str(unpaired); + }) + // This replacement is intentionally implemented without an + // ES2018 negative lookbehind assertion to support runtimes + // which do not yet implement that language feature. + .replace(/(^|[^\ud800-\udbff])([\udc00-\udfff])/g, + function(_, previous, unpaired) { + if (/[\udc00-\udfff]/.test(previous)) { + previous = code_unit_str(previous); + } + + return previous + code_unit_str(unpaired); + }); + } + + function sanitize_all_unpaired_surrogates(tests) { + forEach (tests, + function (test) + { + var sanitized = sanitize_unpaired_surrogates(test.name); + + if (test.name !== sanitized) { + test.name = sanitized; + delete test._structured_clone; + } + }); + } + Tests.prototype.notify_complete = function() { var this_obj = this; var duplicates; @@ -2345,6 +2433,11 @@ policies and contribution forms [3]. if (this.status.status === null) { duplicates = this.find_duplicates(); + // Some transports adhere to UTF-8's restriction on unpaired + // surrogates. Sanitize the titles so that the results can be + // consistently sent via all transports. + sanitize_all_unpaired_surrogates(this.tests); + // Test names are presumed to be unique within test files--this // allows consumers to use them for identification purposes. // Duplicated names violate this expectation and should therefore @@ -2536,6 +2629,9 @@ policies and contribution forms [3]. Output.prototype.resolve_log = function() { var output_document; + if (this.output_node) { + return; + } if (typeof this.output_document === "function") { output_document = this.output_document.apply(undefined); } else { @@ -2546,7 +2642,7 @@ policies and contribution forms [3]. } var node = output_document.getElementById("log"); if (!node) { - if (!document.readyState == "loading") { + if (output_document.readyState === "loading") { return; } node = output_document.createElementNS("http://www.w3.org/1999/xhtml", "div"); @@ -2583,11 +2679,11 @@ policies and contribution forms [3]. if (this.phase < this.STARTED) { this.init(); } - if (!this.enabled) { + if (!this.enabled || this.phase === this.COMPLETE) { return; } + this.resolve_log(); if (this.phase < this.HAVE_RESULTS) { - this.resolve_log(); this.phase = this.HAVE_RESULTS; } var done_count = tests.tests.length - tests.num_pending; diff --git a/test/fixtures/wpt/url/META.yml b/test/fixtures/wpt/url/META.yml index 3a789a0d513f0a..094b266b64b61b 100644 --- a/test/fixtures/wpt/url/META.yml +++ b/test/fixtures/wpt/url/META.yml @@ -2,7 +2,6 @@ spec: https://url.spec.whatwg.org/ suggested_reviewers: - mikewest - domenic - - Sebmaster - annevk - GPHemsley - TimothyGu diff --git a/test/fixtures/wpt/url/historical.any.js b/test/fixtures/wpt/url/historical.any.js index c3797ad263850c..407e118f3a05f9 100644 --- a/test/fixtures/wpt/url/historical.any.js +++ b/test/fixtures/wpt/url/historical.any.js @@ -1,7 +1,9 @@ -test(function() { - assert_false("searchParams" in self.location, - "location object should not have a searchParams attribute"); -}, "searchParams on location object"); +if (self.location) { + test(function() { + assert_false("searchParams" in self.location, + "location object should not have a searchParams attribute"); + }, "searchParams on location object"); +} if(self.GLOBAL.isWindow()) { test(() => { diff --git a/test/fixtures/wpt/url/resources/urltestdata.json b/test/fixtures/wpt/url/resources/urltestdata.json index 26b8ea2e0bc9a1..bf4e2a7833d17f 100644 --- a/test/fixtures/wpt/url/resources/urltestdata.json +++ b/test/fixtures/wpt/url/resources/urltestdata.json @@ -4633,6 +4633,22 @@ "search": "", "hash": "" }, + "# unknown scheme with non-URL characters in the path", + { + "input": "wow:\uFFFF", + "base": "about:blank", + "href": "wow:%EF%BF%BF", + "origin": "null", + "protocol": "wow:", + "username": "", + "password": "", + "host": "", + "hostname": "", + "port": "", + "pathname": "%EF%BF%BF", + "search": "", + "hash": "" + }, "# Hosts and percent-encoding", { "input": "ftp://example.com%80/", diff --git a/test/fixtures/wpt/url/urlsearchparams-constructor.any.js b/test/fixtures/wpt/url/urlsearchparams-constructor.any.js index 6fff03f00fdddd..398021dde2f35f 100644 --- a/test/fixtures/wpt/url/urlsearchparams-constructor.any.js +++ b/test/fixtures/wpt/url/urlsearchparams-constructor.any.js @@ -14,6 +14,11 @@ test(function() { assert_equals(params.toString(), "") }, "URLSearchParams constructor, no arguments") +test(function () { + var params = new URLSearchParams("?a=b") + assert_equals(params.toString(), "a=b") +}, 'URLSearchParams constructor, remove leading "?"') + test(() => { var params = new URLSearchParams(DOMException); assert_equals(params.toString(), "INDEX_SIZE_ERR=1&DOMSTRING_SIZE_ERR=2&HIERARCHY_REQUEST_ERR=3&WRONG_DOCUMENT_ERR=4&INVALID_CHARACTER_ERR=5&NO_DATA_ALLOWED_ERR=6&NO_MODIFICATION_ALLOWED_ERR=7&NOT_FOUND_ERR=8&NOT_SUPPORTED_ERR=9&INUSE_ATTRIBUTE_ERR=10&INVALID_STATE_ERR=11&SYNTAX_ERR=12&INVALID_MODIFICATION_ERR=13&NAMESPACE_ERR=14&INVALID_ACCESS_ERR=15&VALIDATION_ERR=16&TYPE_MISMATCH_ERR=17&SECURITY_ERR=18&NETWORK_ERR=19&ABORT_ERR=20&URL_MISMATCH_ERR=21"A_EXCEEDED_ERR=22&TIMEOUT_ERR=23&INVALID_NODE_TYPE_ERR=24&DATA_CLONE_ERR=25") diff --git a/test/fixtures/wpt/versions.json b/test/fixtures/wpt/versions.json index 7bd65d90237370..0ca53d9103651f 100644 --- a/test/fixtures/wpt/versions.json +++ b/test/fixtures/wpt/versions.json @@ -4,15 +4,15 @@ "path": "console" }, "encoding": { - "commit": "a093a659ed118112138f8a1ffba97a66c1ea8235", + "commit": "7287608f90f6b9530635d10086fd2ab386faab38", "path": "encoding" }, "url": { - "commit": "75b0f336c50105c6fea47ad253d57219dfa744d3", + "commit": "418f7fabebeeb642e79e05b48ffde1a601c7e058", "path": "url" }, "resources": { - "commit": "679a364421ce3704289df21e1ff985c14b360981", + "commit": "e1fddfbf801e7cce9cac5645e992194e4059ef56", "path": "resources" }, "interfaces": { diff --git a/test/internet/test-doctool-versions.js b/test/internet/test-doctool-versions.js new file mode 100644 index 00000000000000..8bb4f81c795d95 --- /dev/null +++ b/test/internet/test-doctool-versions.js @@ -0,0 +1,59 @@ +'use strict'; + +require('../common'); +const assert = require('assert'); +const util = require('util'); +const { versions } = require('../../tools/doc/versions.js'); + +// At the time of writing these are the minimum expected versions. +// New versions of Node.js do not have to be explicitly added here. +const expected = [ + '12.x', + '11.x', + '10.x', + '9.x', + '8.x', + '7.x', + '6.x', + '5.x', + '4.x', + '0.12.x', + '0.10.x', +]; + +async function test() { + const vers = await versions(); + // Coherence checks for each returned version. + for (const version of vers) { + const tested = util.inspect(version); + const parts = version.num.split('.'); + const expectedLength = parts[0] === '0' ? 3 : 2; + assert.strictEqual(parts.length, expectedLength, + `'num' from ${tested} should be '.x'.`); + assert.strictEqual(parts[parts.length - 1], 'x', + `'num' from ${tested} doesn't end in '.x'.`); + const isEvenRelease = Number.parseInt(parts[expectedLength - 2]) % 2 === 0; + const hasLtsProperty = version.hasOwnProperty('lts'); + if (hasLtsProperty) { + // Odd-numbered versions of Node.js are never LTS. + assert.ok(isEvenRelease, `${tested} should not be an 'lts' release.`); + assert.ok(version.lts, `'lts' from ${tested} should 'true'.`); + } + } + + // Check that the minimum number of versions were returned. + // Later versions are allowed, but not checked for here (they were checked + // above). + // Also check for the previous semver major -- From master this will be the + // most recent major release. + const thisMajor = Number.parseInt(process.versions.node.split('.')[0]); + const prevMajorString = `${thisMajor - 1}.x`; + if (!expected.includes(prevMajorString)) { + expected.unshift(prevMajorString); + } + for (const version of expected) { + assert.ok(vers.find((x) => x.num === version), + `Did not find entry for '${version}' in ${util.inspect(vers)}`); + } +} +test(); diff --git a/test/node-api/test_policy/binding.c b/test/node-api/test_policy/binding.c new file mode 100644 index 00000000000000..b896da2cba4d84 --- /dev/null +++ b/test/node-api/test_policy/binding.c @@ -0,0 +1,17 @@ +#include +#include "../../js-native-api/common.h" +#include + +static napi_value Method(napi_env env, napi_callback_info info) { + napi_value world; + const char* str = "world"; + size_t str_len = strlen(str); + NAPI_CALL(env, napi_create_string_utf8(env, str, str_len, &world)); + return world; +} + +NAPI_MODULE_INIT() { + napi_property_descriptor desc = DECLARE_NAPI_PROPERTY("hello", Method); + NAPI_CALL(env, napi_define_properties(env, exports, 1, &desc)); + return exports; +} diff --git a/test/node-api/test_policy/binding.gyp b/test/node-api/test_policy/binding.gyp new file mode 100644 index 00000000000000..62381d5e54f22b --- /dev/null +++ b/test/node-api/test_policy/binding.gyp @@ -0,0 +1,8 @@ +{ + "targets": [ + { + "target_name": "binding", + "sources": [ "binding.c" ] + } + ] +} diff --git a/test/node-api/test_policy/test_policy.js b/test/node-api/test_policy/test_policy.js new file mode 100644 index 00000000000000..b3f477567e37ba --- /dev/null +++ b/test/node-api/test_policy/test_policy.js @@ -0,0 +1,65 @@ +'use strict'; +const common = require('../../common'); +if (!common.hasCrypto) + common.skip('missing crypto'); + +const assert = require('assert'); +const tmpdir = require('../../common/tmpdir'); +const { spawnSync } = require('child_process'); +const crypto = require('crypto'); +const fs = require('fs'); +const path = require('path'); +const { pathToFileURL } = require('url'); + +tmpdir.refresh(); + +function hash(algo, body) { + const h = crypto.createHash(algo); + h.update(body); + return h.digest('base64'); +} + +const policyFilepath = path.join(tmpdir.path, 'policy'); + +const depFilepath = require.resolve(`./build/${common.buildType}/binding.node`); +const depURL = pathToFileURL(depFilepath); + +const tmpdirURL = pathToFileURL(tmpdir.path); +if (!tmpdirURL.pathname.endsWith('/')) { + tmpdirURL.pathname += '/'; +} + +const depBody = fs.readFileSync(depURL); +function writePolicy(...resources) { + const manifest = { resources: {} }; + for (const { url, integrity } of resources) { + manifest.resources[url] = { integrity }; + } + fs.writeFileSync(policyFilepath, JSON.stringify(manifest, null, 2)); +} + + +function test(shouldFail, resources) { + writePolicy(...resources); + const { status, stdout, stderr } = spawnSync(process.execPath, [ + '--experimental-policy', + policyFilepath, + depFilepath + ]); + + console.log(stdout.toString(), stderr.toString()); + if (shouldFail) { + assert.notStrictEqual(status, 0); + } else { + assert.strictEqual(status, 0); + } +} + +test(false, [{ + url: depURL, + integrity: `sha256-${hash('sha256', depBody)}` +}]); +test(true, [{ + url: depURL, + integrity: `sha256akjsalkjdlaskjdk-${hash('sha256', depBody)}` +}]); diff --git a/test/parallel/test-child-process-pipe-dataflow.js b/test/parallel/test-child-process-pipe-dataflow.js index 88a31f4ff8429b..abaec73f3ea507 100644 --- a/test/parallel/test-child-process-pipe-dataflow.js +++ b/test/parallel/test-child-process-pipe-dataflow.js @@ -38,16 +38,19 @@ const MB = KB * KB; grep.stdout._handle.readStart = common.mustNotCall(); [cat, grep, wc].forEach((child, index) => { - child.stderr.on('data', (d) => { + const errorHandler = (thing, type) => { // Don't want to assert here, as we might miss error code info. - console.error(`got unexpected data from child #${index}:\n${d}`); - }); - child.on('exit', common.mustCall(function(code) { - assert.strictEqual(code, 0); + console.error(`unexpected ${type} from child #${index}:\n${thing}`); + }; + + child.stderr.on('data', (d) => { errorHandler(d, 'data'); }); + child.on('error', (err) => { errorHandler(err, 'error'); }); + child.on('exit', common.mustCall((code) => { + assert.strictEqual(code, 0, `child ${index} exited with code ${code}`); })); }); - wc.stdout.on('data', common.mustCall(function(data) { + wc.stdout.on('data', common.mustCall((data) => { assert.strictEqual(data.toString().trim(), MB.toString()); })); } diff --git a/test/parallel/test-cluster-call-and-destroy.js b/test/parallel/test-cluster-call-and-destroy.js new file mode 100644 index 00000000000000..76b5c73d9797c7 --- /dev/null +++ b/test/parallel/test-cluster-call-and-destroy.js @@ -0,0 +1,15 @@ +'use strict'; +const common = require('../common'); +const cluster = require('cluster'); +const assert = require('assert'); + +if (cluster.isMaster) { + const worker = cluster.fork(); + worker.on('disconnect', common.mustCall(() => { + assert.strictEqual(worker.isConnected(), false); + worker.destroy(); + })); +} else { + assert.strictEqual(cluster.worker.isConnected(), true); + cluster.worker.disconnect(); +} diff --git a/test/parallel/test-console-table.js b/test/parallel/test-console-table.js index 3a4d6fefbbc8f1..35bd948f11afb0 100644 --- a/test/parallel/test-console-table.js +++ b/test/parallel/test-console-table.js @@ -244,3 +244,17 @@ test([{ a: 1, b: 'Y' }, { a: 'Z', b: 2 }], ` │ 1 │ 'Z' │ 2 │ └─────────┴─────┴─────┘ `); + +{ + const line = '─'.repeat(79); + const header = `${' '.repeat(37)}name${' '.repeat(40)}`; + const name = 'very long long long long long long long long long long long ' + + 'long long long long'; + test([{ name }], ` +┌─────────┬──${line}──┐ +│ (index) │ ${header}│ +├─────────┼──${line}──┤ +│ 0 │ '${name}' │ +└─────────┴──${line}──┘ +`); +} diff --git a/test/parallel/test-crypto-key-objects.js b/test/parallel/test-crypto-key-objects.js index ab9005f8598a73..2a3a3ec2f0bff1 100644 --- a/test/parallel/test-crypto-key-objects.js +++ b/test/parallel/test-crypto-key-objects.js @@ -13,6 +13,7 @@ const { createSecretKey, createPublicKey, createPrivateKey, + KeyObject, randomBytes, publicEncrypt, privateDecrypt @@ -39,6 +40,27 @@ const privateDsa = fixtures.readKey('dsa_private_encrypted_1025.pem', }); } +{ + // Attempting to create a key of a wrong type should throw + const TYPE = 'wrong_type'; + + common.expectsError(() => new KeyObject(TYPE), { + type: TypeError, + code: 'ERR_INVALID_ARG_VALUE', + message: `The argument 'type' is invalid. Received '${TYPE}'` + }); +} + +{ + // Attempting to create a key with non-object handle should throw + common.expectsError(() => new KeyObject('secret', ''), { + type: TypeError, + code: 'ERR_INVALID_ARG_TYPE', + message: + 'The "handle" argument must be of type object. Received type string' + }); +} + { const keybuf = randomBytes(32); const key = createSecretKey(keybuf); diff --git a/test/parallel/test-crypto-keygen.js b/test/parallel/test-crypto-keygen.js index c65f25e172e75c..8c3432e06cb647 100644 --- a/test/parallel/test-crypto-keygen.js +++ b/test/parallel/test-crypto-keygen.js @@ -378,6 +378,30 @@ const sec1EncExp = (cipher) => getRegExpForPEM('EC PRIVATE KEY', cipher); testSignVerify(publicKey, privateKey); })); + // Test async elliptic curve key generation, e.g. for ECDSA, with a SEC1 + // private key with paramEncoding explicit. + generateKeyPair('ec', { + namedCurve: 'prime256v1', + paramEncoding: 'explicit', + publicKeyEncoding: { + type: 'spki', + format: 'pem' + }, + privateKeyEncoding: { + type: 'sec1', + format: 'pem' + } + }, common.mustCall((err, publicKey, privateKey) => { + assert.ifError(err); + + assert.strictEqual(typeof publicKey, 'string'); + assert(spkiExp.test(publicKey)); + assert.strictEqual(typeof privateKey, 'string'); + assert(sec1Exp.test(privateKey)); + + testSignVerify(publicKey, privateKey); + })); + // Do the same with an encrypted private key. generateKeyPair('ec', { namedCurve: 'prime256v1', @@ -409,6 +433,38 @@ const sec1EncExp = (cipher) => getRegExpForPEM('EC PRIVATE KEY', cipher); testSignVerify(publicKey, { key: privateKey, passphrase: 'secret' }); })); + + // Do the same with an encrypted private key with paramEncoding explicit. + generateKeyPair('ec', { + namedCurve: 'prime256v1', + paramEncoding: 'explicit', + publicKeyEncoding: { + type: 'spki', + format: 'pem' + }, + privateKeyEncoding: { + type: 'sec1', + format: 'pem', + cipher: 'aes-128-cbc', + passphrase: 'secret' + } + }, common.mustCall((err, publicKey, privateKey) => { + assert.ifError(err); + + assert.strictEqual(typeof publicKey, 'string'); + assert(spkiExp.test(publicKey)); + assert.strictEqual(typeof privateKey, 'string'); + assert(sec1EncExp('AES-128-CBC').test(privateKey)); + + // Since the private key is encrypted, signing shouldn't work anymore. + common.expectsError(() => testSignVerify(publicKey, privateKey), { + type: TypeError, + code: 'ERR_MISSING_PASSPHRASE', + message: 'Passphrase required for encrypted key' + }); + + testSignVerify(publicKey, { key: privateKey, passphrase: 'secret' }); + })); } { @@ -447,6 +503,42 @@ const sec1EncExp = (cipher) => getRegExpForPEM('EC PRIVATE KEY', cipher); passphrase: 'top secret' }); })); + + // Test async elliptic curve key generation, e.g. for ECDSA, with an encrypted + // private key with paramEncoding explicit. + generateKeyPair('ec', { + namedCurve: 'P-256', + paramEncoding: 'explicit', + publicKeyEncoding: { + type: 'spki', + format: 'pem' + }, + privateKeyEncoding: { + type: 'pkcs8', + format: 'pem', + cipher: 'aes-128-cbc', + passphrase: 'top secret' + } + }, common.mustCall((err, publicKey, privateKey) => { + assert.ifError(err); + + assert.strictEqual(typeof publicKey, 'string'); + assert(spkiExp.test(publicKey)); + assert.strictEqual(typeof privateKey, 'string'); + assert(pkcs8EncExp.test(privateKey)); + + // Since the private key is encrypted, signing shouldn't work anymore. + common.expectsError(() => testSignVerify(publicKey, privateKey), { + type: TypeError, + code: 'ERR_MISSING_PASSPHRASE', + message: 'Passphrase required for encrypted key' + }); + + testSignVerify(publicKey, { + key: privateKey, + passphrase: 'top secret' + }); + })); } // Test invalid parameter encoding. @@ -907,6 +999,20 @@ const sec1EncExp = (cipher) => getRegExpForPEM('EC PRIVATE KEY', cipher); }); } + // Invalid hash value. + for (const hashValue of [123, true, {}, []]) { + common.expectsError(() => { + generateKeyPairSync('rsa-pss', { + modulusLength: 4096, + hash: hashValue + }); + }, { + type: TypeError, + code: 'ERR_INVALID_OPT_VALUE', + message: `The value "${hashValue}" is invalid for option "hash"` + }); + } + // Invalid private key type. for (const type of ['foo', 'spki']) { common.expectsError(() => { @@ -987,3 +1093,39 @@ const sec1EncExp = (cipher) => getRegExpForPEM('EC PRIVATE KEY', cipher); }); } } +{ + // Test RSA-PSS. + common.expectsError( + () => { + generateKeyPair('rsa-pss', { + modulusLength: 512, + saltLength: 16, + hash: 'sha256', + mgf1Hash: undefined + }); + }, + { + type: TypeError, + code: 'ERR_INVALID_CALLBACK', + message: 'Callback must be a function. Received undefined' + } + ); + + for (const mgf1Hash of [null, 0, false, {}, []]) { + common.expectsError( + () => { + generateKeyPair('rsa-pss', { + modulusLength: 512, + saltLength: 16, + hash: 'sha256', + mgf1Hash + }); + }, + { + type: TypeError, + code: 'ERR_INVALID_OPT_VALUE', + message: `The value "${mgf1Hash}" is invalid for option "mgf1Hash"` + } + ); + } +} diff --git a/test/parallel/test-crypto-lazy-transform-writable.js b/test/parallel/test-crypto-lazy-transform-writable.js index 94240321bd755e..000c6693c8ae0d 100644 --- a/test/parallel/test-crypto-lazy-transform-writable.js +++ b/test/parallel/test-crypto-lazy-transform-writable.js @@ -28,7 +28,7 @@ const stream = new OldStream(); stream.pipe(hasher2).on('finish', common.mustCall(function() { const hash = hasher2.read().toString('hex'); - assert.strictEqual(expected, hash); + assert.strictEqual(hash, expected); })); stream.emit('data', Buffer.from('hello')); diff --git a/test/parallel/test-fs-promises.js b/test/parallel/test-fs-promises.js index 28c047e3055a8c..4c3d7346ffa996 100644 --- a/test/parallel/test-fs-promises.js +++ b/test/parallel/test-fs-promises.js @@ -40,8 +40,11 @@ function nextdir() { return `test${++dirc}`; } -// fs.promises should not enumerable. -assert.strictEqual(Object.keys(fs).includes('promises'), true); +// fs.promises should be enumerable. +assert.strictEqual( + Object.prototype.propertyIsEnumerable.call(fs, 'promises'), + true +); { access(__filename, 'r') diff --git a/test/parallel/test-fs-realpath.js b/test/parallel/test-fs-realpath.js index d12c8a28d2a69b..8b3bb689675dc3 100644 --- a/test/parallel/test-fs-realpath.js +++ b/test/parallel/test-fs-realpath.js @@ -89,6 +89,14 @@ function test_simple_error_callback(realpath, realpathSync, cb) { })); } +function test_simple_error_cb_with_null_options(realpath, realpathSync, cb) { + realpath('/this/path/does/not/exist', null, common.mustCall(function(err, s) { + assert(err); + assert(!s); + cb(); + })); +} + function test_simple_relative_symlink(realpath, realpathSync, callback) { console.log('test_simple_relative_symlink'); if (skipSymlinks) { @@ -395,6 +403,7 @@ function test_up_multiple(realpath, realpathSync, cb) { assertEqualPath(realpathSync(abedabeda), abedabeda_real); assertEqualPath(realpathSync(abedabed), abedabed_real); + realpath(abedabeda, function(er, real) { assert.ifError(er); assertEqualPath(abedabeda_real, real); @@ -407,6 +416,48 @@ function test_up_multiple(realpath, realpathSync, cb) { } +// Going up with .. multiple times with options = null +// . +// `-- a/ +// |-- b/ +// | `-- e -> .. +// `-- d -> .. +// realpath(a/b/e/d/a/b/e/d/a) ==> a +function test_up_multiple_with_null_options(realpath, realpathSync, cb) { + console.error('test_up_multiple'); + if (skipSymlinks) { + common.printSkipMessage('symlink test (no privs)'); + return cb(); + } + const tmpdir = require('../common/tmpdir'); + tmpdir.refresh(); + fs.mkdirSync(tmp('a'), 0o755); + fs.mkdirSync(tmp('a/b'), 0o755); + fs.symlinkSync('..', tmp('a/d'), 'dir'); + unlink.push(tmp('a/d')); + fs.symlinkSync('..', tmp('a/b/e'), 'dir'); + unlink.push(tmp('a/b/e')); + + const abedabed = tmp('abedabed'.split('').join('/')); + const abedabed_real = tmp(''); + + const abedabeda = tmp('abedabeda'.split('').join('/')); + const abedabeda_real = tmp('a'); + + assertEqualPath(realpathSync(abedabeda), abedabeda_real); + assertEqualPath(realpathSync(abedabed), abedabed_real); + + realpath(abedabeda, null, function(er, real) { + assert.ifError(er); + assertEqualPath(abedabeda_real, real); + realpath(abedabed, null, function(er, real) { + assert.ifError(er); + assertEqualPath(abedabed_real, real); + cb(); + }); + }); +} + // Absolute symlinks with children. // . // `-- a/ @@ -474,10 +525,19 @@ function test_root(realpath, realpathSync, cb) { }); } +function test_root_with_null_options(realpath, realpathSync, cb) { + realpath('/', null, function(err, result) { + assert.ifError(err); + assertEqualPath(root, result); + cb(); + }); +} + // ---------------------------------------------------------------------------- const tests = [ test_simple_error_callback, + test_simple_error_cb_with_null_options, test_simple_relative_symlink, test_simple_absolute_symlink, test_deep_relative_file_symlink, @@ -491,7 +551,9 @@ const tests = [ test_upone_actual, test_abs_with_kids, test_up_multiple, + test_up_multiple_with_null_options, test_root, + test_root_with_null_options ]; const numtests = tests.length; let testsRun = 0; diff --git a/test/parallel/test-fs-write-sigxfsz.js b/test/parallel/test-fs-write-sigxfsz.js new file mode 100644 index 00000000000000..323312fcb943dc --- /dev/null +++ b/test/parallel/test-fs-write-sigxfsz.js @@ -0,0 +1,32 @@ +// Check that exceeding RLIMIT_FSIZE fails with EFBIG +// rather than terminating the process with SIGXFSZ. +'use strict'; +const common = require('../common'); +const tmpdir = require('../common/tmpdir'); + +const assert = require('assert'); +const child_process = require('child_process'); +const fs = require('fs'); +const path = require('path'); + +if (common.isWindows) + common.skip('no RLIMIT_FSIZE on Windows'); + +if (process.config.variables.node_shared) + common.skip('SIGXFSZ signal handler not installed in shared library mode'); + +if (process.argv[2] === 'child') { + const filename = path.join(tmpdir.path, 'efbig.txt'); + tmpdir.refresh(); + fs.writeFileSync(filename, '.'.repeat(1 << 16)); // Exceeds RLIMIT_FSIZE. +} else { + const cmd = `ulimit -f 1 && '${process.execPath}' '${__filename}' child`; + const result = child_process.spawnSync('/bin/sh', ['-c', cmd]); + const haystack = result.stderr.toString(); + const needle = 'Error: EFBIG: file too large, write'; + const ok = haystack.includes(needle); + if (!ok) console.error(haystack); + assert(ok); + assert.strictEqual(result.status, 1); + assert.strictEqual(result.stdout.toString(), ''); +} diff --git a/test/parallel/test-http-outgoing-message-write-callback.js b/test/parallel/test-http-outgoing-message-write-callback.js index a51d37351d4c9e..3a32285faaff55 100644 --- a/test/parallel/test-http-outgoing-message-write-callback.js +++ b/test/parallel/test-http-outgoing-message-write-callback.js @@ -3,35 +3,37 @@ const common = require('../common'); // This test ensures that the callback of `OutgoingMessage.prototype.write()` is -// called also when writing empty chunks. +// called also when writing empty chunks or when the message has no body. const assert = require('assert'); const http = require('http'); const stream = require('stream'); -const expected = ['a', 'b', '', Buffer.alloc(0), 'c']; -const results = []; +for (const method of ['GET, HEAD']) { + const expected = ['a', 'b', '', Buffer.alloc(0), 'c']; + const results = []; -const writable = new stream.Writable({ - write(chunk, encoding, callback) { - setImmediate(callback); - } -}); + const writable = new stream.Writable({ + write(chunk, encoding, callback) { + callback(); + } + }); -const res = new http.ServerResponse({ - method: 'GET', - httpVersionMajor: 1, - httpVersionMinor: 1 -}); + const res = new http.ServerResponse({ + method: method, + httpVersionMajor: 1, + httpVersionMinor: 1 + }); -res.assignSocket(writable); + res.assignSocket(writable); -for (const chunk of expected) { - res.write(chunk, () => { - results.push(chunk); - }); -} + for (const chunk of expected) { + res.write(chunk, () => { + results.push(chunk); + }); + } -res.end(common.mustCall(() => { - assert.deepStrictEqual(results, expected); -})); + res.end(common.mustCall(() => { + assert.deepStrictEqual(results, expected); + })); +} diff --git a/test/parallel/test-http-server-delete-parser.js b/test/parallel/test-http-server-delete-parser.js index 0c5eea90734170..4215ee2f9df74e 100644 --- a/test/parallel/test-http-server-delete-parser.js +++ b/test/parallel/test-http-server-delete-parser.js @@ -12,13 +12,13 @@ const server = http.createServer(common.mustCall((req, res) => { res.end(); })); -server.listen(1337, '127.0.0.1'); -server.unref(); - -const req = http.request({ - port: 1337, - host: '127.0.0.1', - method: 'GET', -}); +server.listen(0, '127.0.0.1', common.mustCall(() => { + const req = http.request({ + port: server.address().port, + host: '127.0.0.1', + method: 'GET', + }); + req.end(); +})); -req.end(); +server.unref(); diff --git a/test/parallel/test-http-server-unconsume.js b/test/parallel/test-http-server-unconsume.js index c285a53c7ac215..a8a307e53b5a30 100644 --- a/test/parallel/test-http-server-unconsume.js +++ b/test/parallel/test-http-server-unconsume.js @@ -1,34 +1,33 @@ 'use strict'; -require('../common'); +const common = require('../common'); const assert = require('assert'); const http = require('http'); const net = require('net'); -let received = ''; +['on', 'addListener', 'prependListener'].forEach((testFn) => { + let received = ''; -const server = http.createServer(function(req, res) { - res.writeHead(200); - res.end(); + const server = http.createServer(function(req, res) { + res.writeHead(200); + res.end(); - req.socket.on('data', function(data) { - received += data; - }); + req.socket[testFn]('data', function(data) { + received += data; + }); - assert.strictEqual(req.socket.on, req.socket.addListener); - assert.strictEqual(req.socket.prependListener, - net.Socket.prototype.prependListener); + server.close(); + }).listen(0, function() { + const socket = net.connect(this.address().port, function() { + socket.write('PUT / HTTP/1.1\r\n\r\n'); - server.close(); -}).listen(0, function() { - const socket = net.connect(this.address().port, function() { - socket.write('PUT / HTTP/1.1\r\n\r\n'); + socket.once('data', function() { + socket.end('hello world'); + }); - socket.once('data', function() { - socket.end('hello world'); + socket.on('end', common.mustCall(() => { + assert.strictEqual(received, 'hello world', + `failed for socket.${testFn}`); + })); }); }); }); - -process.on('exit', function() { - assert.strictEqual(received, 'hello world'); -}); diff --git a/test/parallel/test-http-timeout-flag.js b/test/parallel/test-http-timeout-flag.js new file mode 100644 index 00000000000000..f816ce2f0a4f4b --- /dev/null +++ b/test/parallel/test-http-timeout-flag.js @@ -0,0 +1,41 @@ +'use strict'; +const common = require('../common'); + +if (!common.hasCrypto) + common.skip('missing crypto'); + +const fixtures = require('../common/fixtures'); +const http = require('http'); +const https = require('https'); +const http2 = require('http2'); +const assert = require('assert'); +const { spawnSync } = require('child_process'); + +// Make sure the defaults are correct. +const servers = [ + http.createServer(), + https.createServer({ + key: fixtures.readKey('agent1-key.pem'), + cert: fixtures.readKey('agent1-cert.pem') + }), + http2.createServer() +]; + +for (const server of servers) { + assert.strictEqual(server.timeout, 120000); + server.close(); +} + +// Ensure that command line flag overrides the default timeout. +const child1 = spawnSync(process.execPath, ['--http-server-default-timeout=10', + '-p', 'http.createServer().timeout' +]); +assert.strictEqual(+child1.stdout.toString().trim(), 10); + +// Ensure that the flag is whitelisted for NODE_OPTIONS. +const env = Object.assign({}, process.env, { + NODE_OPTIONS: '--http-server-default-timeout=10' +}); +const child2 = spawnSync(process.execPath, + [ '-p', 'http.createServer().timeout'], { env }); +assert.strictEqual(+child2.stdout.toString().trim(), 10); diff --git a/test/parallel/test-http-upgrade-client.js b/test/parallel/test-http-upgrade-client.js index c637324a53d801..d18ac273d6dab1 100644 --- a/test/parallel/test-http-upgrade-client.js +++ b/test/parallel/test-http-upgrade-client.js @@ -31,6 +31,8 @@ const http = require('http'); const net = require('net'); const Countdown = require('../common/countdown'); +const expectedRecvData = 'nurtzo'; + // Create a TCP server const srv = net.createServer(function(c) { c.on('data', function(d) { @@ -39,7 +41,7 @@ const srv = net.createServer(function(c) { c.write('connection: upgrade\r\n'); c.write('upgrade: websocket\r\n'); c.write('\r\n'); - c.write('nurtzo'); + c.write(expectedRecvData); }); c.on('end', function() { @@ -77,7 +79,7 @@ srv.listen(0, '127.0.0.1', common.mustCall(function() { }); socket.on('close', common.mustCall(function() { - assert.strictEqual(recvData.toString(), 'nurtzo'); + assert.strictEqual(recvData.toString(), expectedRecvData); })); console.log(res.headers); @@ -86,8 +88,7 @@ srv.listen(0, '127.0.0.1', common.mustCall(function() { connection: 'upgrade', upgrade: 'websocket' }; - assert.deepStrictEqual(expectedHeaders, res.headers); - + assert.deepStrictEqual(res.headers, expectedHeaders); socket.end(); countdown.dec(); })); diff --git a/test/parallel/test-http2-large-write-destroy.js b/test/parallel/test-http2-large-write-destroy.js index 24c0a055cc943f..b59c66bb04755b 100644 --- a/test/parallel/test-http2-large-write-destroy.js +++ b/test/parallel/test-http2-large-write-destroy.js @@ -32,6 +32,7 @@ server.listen(0, common.mustCall(() => { const req = client.request({ ':path': '/' }); req.end(); + req.resume(); // Otherwise close won't be emitted if there's pending data. req.on('close', common.mustCall(() => { client.close(); diff --git a/test/parallel/test-http2-max-session-memory-leak.js b/test/parallel/test-http2-max-session-memory-leak.js new file mode 100644 index 00000000000000..b066ca80bc5eab --- /dev/null +++ b/test/parallel/test-http2-max-session-memory-leak.js @@ -0,0 +1,46 @@ +'use strict'; +const common = require('../common'); +if (!common.hasCrypto) + common.skip('missing crypto'); +const http2 = require('http2'); + +// Regression test for https://github.com/nodejs/node/issues/27416. +// Check that received data is accounted for correctly in the maxSessionMemory +// mechanism. + +const bodyLength = 8192; +const maxSessionMemory = 1; // 1 MB +const requestCount = 1000; + +const server = http2.createServer({ maxSessionMemory }); +server.on('stream', (stream) => { + stream.respond(); + stream.end(); +}); + +server.listen(common.mustCall(() => { + const client = http2.connect(`http://localhost:${server.address().port}`, { + maxSessionMemory + }); + + function request() { + return new Promise((resolve, reject) => { + const stream = client.request({ + ':method': 'POST', + 'content-length': bodyLength + }); + stream.on('error', reject); + stream.on('response', resolve); + stream.end('a'.repeat(bodyLength)); + }); + } + + (async () => { + for (let i = 0; i < requestCount; i++) { + await request(); + } + + client.close(); + server.close(); + })().then(common.mustCall()); +})); diff --git a/test/parallel/test-http2-multiheaders-raw.js b/test/parallel/test-http2-multiheaders-raw.js index da9aa3a68eaa51..32bf9d05433d42 100644 --- a/test/parallel/test-http2-multiheaders-raw.js +++ b/test/parallel/test-http2-multiheaders-raw.js @@ -34,7 +34,7 @@ server.on('stream', common.mustCall((stream, headers, flags, rawHeaders) => { 'foo, bar, baz' ]; - assert.deepStrictEqual(expected, rawHeaders); + assert.deepStrictEqual(rawHeaders, expected); stream.respond(src); stream.end(); })); diff --git a/test/parallel/test-http2-origin.js b/test/parallel/test-http2-origin.js index d0d8c81f801a88..7aa0a16e35f22e 100644 --- a/test/parallel/test-http2-origin.js +++ b/test/parallel/test-http2-origin.js @@ -96,7 +96,7 @@ const ca = readKey('fake-startcom-root-cert.pem', 'binary'); client.on('origin', mustCall((origins) => { const check = checks.shift(); originSet.push(...check); - deepStrictEqual(originSet, client.originSet); + deepStrictEqual(client.originSet, originSet); deepStrictEqual(origins, check); countdown.dec(); }, 2)); @@ -121,7 +121,7 @@ const ca = readKey('fake-startcom-root-cert.pem', 'binary'); client.on('origin', mustCall((origins) => { originSet.push(...check); - deepStrictEqual(originSet, client.originSet); + deepStrictEqual(client.originSet, originSet); deepStrictEqual(origins, check); client.close(); server.close(); @@ -148,11 +148,11 @@ const ca = readKey('fake-startcom-root-cert.pem', 'binary'); const client = connect(origin, { ca }); client.on('origin', mustCall((origins) => { - deepStrictEqual([origin, 'https://foo.org'], client.originSet); + deepStrictEqual(client.originSet, [origin, 'https://foo.org']); const req = client.request({ ':authority': 'foo.org' }); req.on('response', mustCall((headers) => { - strictEqual(421, headers[':status']); - deepStrictEqual([origin], client.originSet); + strictEqual(headers[':status'], 421); + deepStrictEqual(client.originSet, [origin]); })); req.resume(); req.on('close', mustCall(() => { diff --git a/test/parallel/test-http2-server-startup.js b/test/parallel/test-http2-server-startup.js index 4ebcc21c27aa8e..c94abd2c22eddb 100644 --- a/test/parallel/test-http2-server-startup.js +++ b/test/parallel/test-http2-server-startup.js @@ -10,6 +10,7 @@ const commonFixtures = require('../common/fixtures'); if (!common.hasCrypto) common.skip('missing crypto'); +const assert = require('assert'); const http2 = require('http2'); const tls = require('tls'); const net = require('net'); @@ -48,6 +49,25 @@ server.on('error', common.mustNotCall()); })); } +// Test that `http2.createServer()` supports `net.Server` options. +{ + const server = http2.createServer({ allowHalfOpen: true }); + + server.on('connection', common.mustCall((socket) => { + assert.strictEqual(socket.allowHalfOpen, true); + socket.end(); + server.close(); + })); + + assert.strictEqual(server.allowHalfOpen, true); + + server.listen(0, common.mustCall(() => { + const port = server.address().port; + const socket = net.connect(port, common.mustCall()); + socket.resume(); + })); +} + // Test the secure server socket timeout. { let client; @@ -67,3 +87,29 @@ server.on('error', common.mustNotCall()); }, common.mustCall()); })); } + +// Test that `http2.createSecureServer()` supports `net.Server` options. +{ + const server = http2.createSecureServer({ + allowHalfOpen: true, + ...options + }); + + server.on('secureConnection', common.mustCall((socket) => { + assert.strictEqual(socket.allowHalfOpen, true); + socket.end(); + server.close(); + })); + + assert.strictEqual(server.allowHalfOpen, true); + + server.listen(0, common.mustCall(() => { + const port = server.address().port; + const socket = tls.connect({ + port: port, + rejectUnauthorized: false, + ALPNProtocols: ['h2'] + }, common.mustCall()); + socket.resume(); + })); +} diff --git a/test/parallel/test-https-agent-additional-options.js b/test/parallel/test-https-agent-additional-options.js index a04ef7461d033f..99fcd0126ab1a0 100644 --- a/test/parallel/test-https-agent-additional-options.js +++ b/test/parallel/test-https-agent-additional-options.js @@ -1,4 +1,3 @@ -// Flags: --tls-min-v1.1 'use strict'; const common = require('../common'); if (!common.hasCrypto) @@ -12,10 +11,11 @@ const fixtures = require('../common/fixtures'); const options = { key: fixtures.readKey('agent1-key.pem'), cert: fixtures.readKey('agent1-cert.pem'), - ca: fixtures.readKey('ca1-cert.pem') + ca: fixtures.readKey('ca1-cert.pem'), + minVersion: 'TLSv1.1', }; -const server = https.Server(options, function(req, res) { +const server = https.Server(options, (req, res) => { res.writeHead(200); res.end('hello world\n'); }); @@ -39,50 +39,39 @@ const updatedValues = new Map([ ['sessionIdContext', 'sessionIdContext'], ]); +let value; function variations(iter, port, cb) { - const { done, value } = iter.next(); - if (done) { - return common.mustCall(cb); - } else { - const [key, val] = value; - return common.mustCall(function(res) { - res.resume(); - https.globalAgent.once('free', common.mustCall(function() { - https.get( - Object.assign({}, getBaseOptions(port), { [key]: val }), - variations(iter, port, cb) + return common.mustCall((res) => { + res.resume(); + https.globalAgent.once('free', common.mustCall(() => { + // Verify that the most recent connection is in the freeSockets pool. + const keys = Object.keys(https.globalAgent.freeSockets); + if (value) { + assert.ok( + keys.some((val) => val.startsWith(value.toString() + ':') || + val.endsWith(':' + value.toString()) || + val.includes(':' + value.toString() + ':')), + `missing value: ${value.toString()} in ${keys}` ); - })); - }); - } -} + } + const next = iter.next(); -server.listen(0, common.mustCall(function() { - const port = this.address().port; - const globalAgent = https.globalAgent; - globalAgent.keepAlive = true; - https.get(getBaseOptions(port), variations( - updatedValues.entries(), - port, - common.mustCall(function(res) { - res.resume(); - globalAgent.once('free', common.mustCall(function() { - // Verify that different keep-alived connections are created - // for the base call and each variation - const keys = Object.keys(globalAgent.freeSockets); - assert.strictEqual(keys.length, 1 + updatedValues.size); - let i = 1; - for (const [, value] of updatedValues) { - assert.ok( - keys[i].startsWith(value.toString() + ':') || - keys[i].endsWith(':' + value.toString()) || - keys[i].includes(':' + value.toString() + ':') - ); - i++; - } - globalAgent.destroy(); + if (next.done) { + https.globalAgent.destroy(); server.close(); - })); - }) - )); + } else { + // Save `value` for check the next time. + value = next.value.val; + const [key, val] = next.value; + https.get(Object.assign({}, getBaseOptions(port), { [key]: val }), + variations(iter, port, cb)); + } + })); + }); +} + +server.listen(0, common.mustCall(() => { + const port = server.address().port; + https.globalAgent.keepAlive = true; + https.get(getBaseOptions(port), variations(updatedValues.entries(), port)); })); diff --git a/test/parallel/test-net-binary.js b/test/parallel/test-net-binary.js index b8dbc0e8d70e8f..cf8715411d0f7f 100644 --- a/test/parallel/test-net-binary.js +++ b/test/parallel/test-net-binary.js @@ -76,7 +76,7 @@ echoServer.on('listening', function() { }); process.on('exit', function() { - assert.strictEqual(2 * 256, recv.length); + assert.strictEqual(recv.length, 2 * 256); const a = recv.split(''); diff --git a/test/parallel/test-path-resolve.js b/test/parallel/test-path-resolve.js index e9e6d83ff54007..ed3c7c4c5a0113 100644 --- a/test/parallel/test-path-resolve.js +++ b/test/parallel/test-path-resolve.js @@ -69,3 +69,12 @@ if (common.isWindows) { const resolvedPath = spawnResult.stdout.toString().trim(); assert.strictEqual(resolvedPath.toLowerCase(), process.cwd().toLowerCase()); } + +if (!common.isWindows) { + // Test handling relative paths to be safe when process.cwd() fails. + process.cwd = () => ''; + assert.strictEqual(process.cwd(), ''); + const resolved = path.resolve(); + const expected = '.'; + assert.strictEqual(resolved, expected); +} diff --git a/test/parallel/test-stream-readable-setEncoding-existing-buffers.js b/test/parallel/test-stream-readable-setEncoding-existing-buffers.js new file mode 100644 index 00000000000000..eb75260bacfc45 --- /dev/null +++ b/test/parallel/test-stream-readable-setEncoding-existing-buffers.js @@ -0,0 +1,60 @@ +'use strict'; +require('../common'); +const { Readable } = require('stream'); +const assert = require('assert'); + +{ + // Call .setEncoding() while there are bytes already in the buffer. + const r = new Readable({ read() {} }); + + r.push(Buffer.from('a')); + r.push(Buffer.from('b')); + + r.setEncoding('utf8'); + const chunks = []; + r.on('data', (chunk) => chunks.push(chunk)); + + process.nextTick(() => { + assert.deepStrictEqual(chunks, ['ab']); + }); +} + +{ + // Call .setEncoding() while the buffer contains a complete, + // but chunked character. + const r = new Readable({ read() {} }); + + r.push(Buffer.from([0xf0])); + r.push(Buffer.from([0x9f])); + r.push(Buffer.from([0x8e])); + r.push(Buffer.from([0x89])); + + r.setEncoding('utf8'); + const chunks = []; + r.on('data', (chunk) => chunks.push(chunk)); + + process.nextTick(() => { + assert.deepStrictEqual(chunks, ['🎉']); + }); +} + +{ + // Call .setEncoding() while the buffer contains an incomplete character, + // and finish the character later. + const r = new Readable({ read() {} }); + + r.push(Buffer.from([0xf0])); + r.push(Buffer.from([0x9f])); + + r.setEncoding('utf8'); + + r.push(Buffer.from([0x8e])); + r.push(Buffer.from([0x89])); + + const chunks = []; + r.on('data', (chunk) => chunks.push(chunk)); + + process.nextTick(() => { + assert.deepStrictEqual(chunks, ['🎉']); + }); +} diff --git a/test/parallel/test-stream-readable-unshift.js b/test/parallel/test-stream-readable-unshift.js new file mode 100644 index 00000000000000..d574b7d0465b76 --- /dev/null +++ b/test/parallel/test-stream-readable-unshift.js @@ -0,0 +1,187 @@ +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const { Readable } = require('stream'); + +{ + // Check that strings are saved as Buffer + const readable = new Readable({ read() {} }); + + const string = 'abc'; + + readable.on('data', common.mustCall((chunk) => { + assert(Buffer.isBuffer(chunk)); + assert.strictEqual(chunk.toString('utf8'), string); + }, 1)); + + readable.unshift(string); + +} + +{ + // Check that data goes at the beginning + const readable = new Readable({ read() {} }); + const unshift = 'front'; + const push = 'back'; + + const expected = [unshift, push]; + readable.on('data', common.mustCall((chunk) => { + assert.strictEqual(chunk.toString('utf8'), expected.shift()); + }, 2)); + + + readable.push(push); + readable.unshift(unshift); +} + +{ + // Check that buffer is saved with correct encoding + const readable = new Readable({ read() {} }); + + const encoding = 'base64'; + const string = Buffer.from('abc').toString(encoding); + + readable.on('data', common.mustCall((chunk) => { + assert.strictEqual(chunk.toString(encoding), string); + }, 1)); + + readable.unshift(string, encoding); + +} + +{ + + const streamEncoding = 'base64'; + + function checkEncoding(readable) { + + // chunk encodings + const encodings = ['utf8', 'binary', 'hex', 'base64']; + const expected = []; + + readable.on('data', common.mustCall((chunk) => { + const { encoding, string } = expected.pop(); + assert.strictEqual(chunk.toString(encoding), string); + }, encodings.length)); + + for (const encoding of encodings) { + const string = 'abc'; + + // If encoding is the same as the state.encoding the string is + // saved as is + const expect = encoding !== streamEncoding ? + Buffer.from(string, encoding).toString(streamEncoding) : string; + + expected.push({ encoding, string: expect }); + + readable.unshift(string, encoding); + } + } + + const r1 = new Readable({ read() {} }); + r1.setEncoding(streamEncoding); + checkEncoding(r1); + + const r2 = new Readable({ read() {}, encoding: streamEncoding }); + checkEncoding(r2); + +} + +{ + // Both .push & .unshift should have the same behaviour + // When setting an encoding, each chunk should be emitted with that encoding + const encoding = 'base64'; + + function checkEncoding(readable) { + const string = 'abc'; + readable.on('data', common.mustCall((chunk) => { + assert.strictEqual(chunk, Buffer.from(string).toString(encoding)); + }, 2)); + + readable.push(string); + readable.unshift(string); + } + + const r1 = new Readable({ read() {} }); + r1.setEncoding(encoding); + checkEncoding(r1); + + const r2 = new Readable({ read() {}, encoding }); + checkEncoding(r2); + +} + +{ + // Check that error is thrown for invalid chunks + + const readable = new Readable({ read() {} }); + function checkError(fn) { + common.expectsError(fn, { + code: 'ERR_INVALID_ARG_TYPE', + type: TypeError + }); + } + + checkError(() => readable.unshift([])); + checkError(() => readable.unshift({})); + checkError(() => readable.unshift(0)); + +} + +{ + // Check that ObjectMode works + const readable = new Readable({ objectMode: true, read() {} }); + + const chunks = ['a', 1, {}, []]; + + readable.on('data', common.mustCall((chunk) => { + assert.strictEqual(chunk, chunks.pop()); + }, chunks.length)); + + for (const chunk of chunks) { + readable.unshift(chunk); + } +} + +{ + + // Should not throw: https://github.com/nodejs/node/issues/27192 + const highWaterMark = 50; + class ArrayReader extends Readable { + constructor(opt) { + super({ highWaterMark }); + // The error happened only when pushing above hwm + this.buffer = new Array(highWaterMark * 2).fill(0).map(String); + } + _read(size) { + while (this.buffer.length) { + const chunk = this.buffer.shift(); + if (!this.buffer.length) { + this.push(chunk); + this.push(null); + return true; + } + if (!this.push(chunk)) + return; + } + } + } + + function onRead() { + while (null !== (stream.read())) { + // Remove the 'readable' listener before unshifting + stream.removeListener('readable', onRead); + stream.unshift('a'); + stream.on('data', (chunk) => { + console.log(chunk.length); + }); + break; + } + } + + const stream = new ArrayReader(); + stream.once('readable', common.mustCall(onRead)); + stream.on('end', common.mustCall(() => {})); + +} diff --git a/test/parallel/test-tls-connect-hints-option.js b/test/parallel/test-tls-connect-hints-option.js new file mode 100644 index 00000000000000..fd155c9659ad0c --- /dev/null +++ b/test/parallel/test-tls-connect-hints-option.js @@ -0,0 +1,26 @@ +'use strict'; + +const common = require('../common'); + +// This test verifies that `tls.connect()` honors the `hints` option. + +if (!common.hasCrypto) + common.skip('missing crypto'); + +const assert = require('assert'); +const dns = require('dns'); +const tls = require('tls'); + +const hints = 512; + +assert.notStrictEqual(hints, dns.ADDRCONFIG); +assert.notStrictEqual(hints, dns.V4MAPPED); +assert.notStrictEqual(hints, dns.ADDRCONFIG | dns.V4MAPPED); + +tls.connect({ + lookup: common.mustCall((host, options) => { + assert.strictEqual(host, 'localhost'); + assert.deepStrictEqual(options, { family: undefined, hints }); + }), + hints +}); diff --git a/test/parallel/test-tls-enable-trace-cli.js b/test/parallel/test-tls-enable-trace-cli.js index d41334a0f51d63..4d3065e757fce6 100644 --- a/test/parallel/test-tls-enable-trace-cli.js +++ b/test/parallel/test-tls-enable-trace-cli.js @@ -36,8 +36,8 @@ child.on('close', common.mustCall((code, signal) => { assert.strictEqual(signal, null); assert.strictEqual(stdout.trim(), ''); assert(/Warning: Enabling --trace-tls can expose sensitive/.test(stderr)); + assert(/Sent Record/.test(stderr)); assert(/Received Record/.test(stderr)); - assert(/ClientHello/.test(stderr)); })); function test() { @@ -55,12 +55,14 @@ function test() { key: keys.agent6.key }, }, common.mustCall((err, pair, cleanup) => { - if (err) { - console.error(err); - console.error(err.opensslErrorStack); - console.error(err.reason); - assert(err); + if (pair.server.err) { + console.trace('server', pair.server.err); } + if (pair.client.err) { + console.trace('client', pair.client.err); + } + assert.ifError(pair.server.err); + assert.ifError(pair.client.err); return cleanup(); })); diff --git a/test/parallel/test-tls-sni-servername.js b/test/parallel/test-tls-sni-servername.js new file mode 100644 index 00000000000000..2c5785df5426c9 --- /dev/null +++ b/test/parallel/test-tls-sni-servername.js @@ -0,0 +1,56 @@ +'use strict'; +const common = require('../common'); +if (!common.hasCrypto) + common.skip('missing crypto'); + +const assert = require('assert'); +const tls = require('tls'); + +// We could get the `tlsSocket.servername` even if the event of "tlsClientError" +// is emitted. + +const serverOptions = { + requestCert: true, + rejectUnauthorized: false, + SNICallback: function(servername, callback) { + if (servername === 'c.another.com') { + callback(null, {}); + } else { + callback(new Error('Invalid SNI context'), null); + } + } +}; + +function test(options) { + const server = tls.createServer(serverOptions, common.mustNotCall()); + + server.on('tlsClientError', common.mustCall((err, socket) => { + assert.strictEqual(err.message, 'Invalid SNI context'); + // The `servername` should match. + assert.strictEqual(socket.servername, options.servername); + })); + + server.listen(0, () => { + options.port = server.address().port; + const client = tls.connect(options, common.mustNotCall()); + + client.on('error', common.mustCall((err) => { + assert.strictEqual(err.message, 'Client network socket' + + ' disconnected before secure TLS connection was established'); + })); + + client.on('close', common.mustCall(() => server.close())); + }); +} + +test({ + port: undefined, + servername: 'c.another.com', + rejectUnauthorized: false +}); + +test({ + port: undefined, + servername: 'c.wrong.com', + rejectUnauthorized: false +}); diff --git a/test/parallel/test-util-inspect.js b/test/parallel/test-util-inspect.js index cfd962097810ce..6646a56294872b 100644 --- a/test/parallel/test-util-inspect.js +++ b/test/parallel/test-util-inspect.js @@ -339,6 +339,15 @@ assert.strictEqual( const value = {}; value.a = value; assert.strictEqual(util.inspect(value), '{ a: [Circular] }'); + const getterFn = { + get one() { + return null; + } + }; + assert.strictEqual( + util.inspect(getterFn, { getters: true }), + '{ one: [Getter: null] }' + ); } // Array with dynamic properties. @@ -511,6 +520,10 @@ assert.strictEqual(util.inspect(-5e-324), '-5e-324'); util.inspect(a, { maxArrayLength: 4 }), "[ 'foo', <1 empty item>, 'baz', <97 empty items>, ... 1 more item ]" ); + // test 4 special case + assert.strictEqual(util.inspect(a, { + maxArrayLength: 2 + }), "[ 'foo', <1 empty item>, ... 99 more items ]"); } // Test for Array constructor in different context. @@ -852,6 +865,13 @@ util.inspect({ hasOwnProperty: null }); util.inspect(subject, { customInspectOptions: true, seen: null }); } +{ + const subject = { [util.inspect.custom]: common.mustCall((depth) => { + assert.strictEqual(depth, null); + }) }; + util.inspect(subject, { depth: null }); +} + { // Returning `this` from a custom inspection function works. const subject = { a: 123, [util.inspect.custom]() { return this; } }; @@ -895,6 +915,10 @@ assert.strictEqual( // Test boxed primitives output the correct values. assert.strictEqual(util.inspect(new String('test')), "[String: 'test']"); +assert.strictEqual( + util.inspect(new String('test'), { colors: true }), + "\u001b[32m[String: 'test']\u001b[39m" +); assert.strictEqual( util.inspect(Object(Symbol('test'))), '[Symbol: Symbol(test)]' diff --git a/test/parallel/test-v8-stats.js b/test/parallel/test-v8-stats.js index 83b515d3fd56ec..3e2eadef1711ae 100644 --- a/test/parallel/test-v8-stats.js +++ b/test/parallel/test-v8-stats.js @@ -8,6 +8,8 @@ const keys = [ 'does_zap_garbage', 'heap_size_limit', 'malloced_memory', + 'number_of_detached_contexts', + 'number_of_native_contexts', 'peak_malloced_memory', 'total_available_size', 'total_heap_size', diff --git a/test/parallel/test-vm-module-basic.js b/test/parallel/test-vm-module-basic.js index 8cf687c2bb23a8..caf4be328d5873 100644 --- a/test/parallel/test-vm-module-basic.js +++ b/test/parallel/test-vm-module-basic.js @@ -5,6 +5,7 @@ const common = require('../common'); const assert = require('assert'); const { SourceTextModule, createContext } = require('vm'); +const util = require('util'); (async function test1() { const context = createContext({ @@ -63,3 +64,21 @@ const { SourceTextModule, createContext } = require('vm'); const m3 = new SourceTextModule('3', { context: context2 }); assert.strictEqual(m3.url, 'vm:module(0)'); })(); + +// Check inspection of the instance +{ + const context = createContext({ foo: 'bar' }); + const m = new SourceTextModule('1', { context }); + + assert.strictEqual( + util.inspect(m), + "SourceTextModule {\n status: 'uninstantiated',\n linkingStatus:" + + " 'unlinked',\n url: 'vm:module(0)',\n context: { foo: 'bar' }\n}" + ); + assert.strictEqual( + m[util.inspect.custom].call(Object.create(null)), + 'SourceTextModule {\n status: undefined,\n linkingStatus: undefined,' + + '\n url: undefined,\n context: undefined\n}' + ); + assert.strictEqual(util.inspect(m, { depth: -1 }), '[SourceTextModule]'); +} diff --git a/test/parallel/test-worker-debug.js b/test/parallel/test-worker-debug.js index 8726d2a031ee32..4906f623e85ef1 100644 --- a/test/parallel/test-worker-debug.js +++ b/test/parallel/test-worker-debug.js @@ -206,6 +206,52 @@ async function testTwoWorkers(session, post) { await Promise.all([worker1Exited, worker2Exited]); } +async function testWaitForDisconnectInWorker(session, post) { + console.log('Test NodeRuntime.waitForDisconnect in worker'); + + const sessionWithoutWaiting = new Session(); + sessionWithoutWaiting.connect(); + const sessionWithoutWaitingPost = doPost.bind(null, sessionWithoutWaiting); + + await sessionWithoutWaitingPost('NodeWorker.enable', { + waitForDebuggerOnStart: true + }); + await post('NodeWorker.enable', { waitForDebuggerOnStart: true }); + + const attached = [ + waitForWorkerAttach(session), + waitForWorkerAttach(sessionWithoutWaiting) + ]; + + let worker = null; + const exitPromise = runWorker(2, (w) => worker = w); + + const [{ sessionId: sessionId1 }, { sessionId: sessionId2 }] = + await Promise.all(attached); + + const workerSession1 = new WorkerSession(session, sessionId1); + const workerSession2 = new WorkerSession(sessionWithoutWaiting, sessionId2); + + await workerSession2.post('Runtime.enable'); + await workerSession1.post('Runtime.enable'); + await workerSession1.post('NodeRuntime.notifyWhenWaitingForDisconnect', { + enabled: true + }); + await workerSession1.post('Runtime.runIfWaitingForDebugger'); + + worker.postMessage('resume'); + + await waitForEvent(workerSession1, 'NodeRuntime.waitingForDisconnect'); + post('NodeWorker.detach', { sessionId: sessionId1 }); + await waitForEvent(workerSession2, 'Runtime.executionContextDestroyed'); + + await exitPromise; + + await post('NodeWorker.disable'); + await sessionWithoutWaitingPost('NodeWorker.disable'); + sessionWithoutWaiting.disconnect(); +} + async function test() { const session = new Session(); session.connect(); @@ -219,6 +265,7 @@ async function test() { await testNoWaitOnStart(session, post); await testTwoWorkers(session, post); + await testWaitForDisconnectInWorker(session, post); session.disconnect(); console.log('Test done'); diff --git a/test/parallel/test-worker-message-not-serializable.js b/test/parallel/test-worker-message-not-serializable.js new file mode 100644 index 00000000000000..3753c7de6cbdf9 --- /dev/null +++ b/test/parallel/test-worker-message-not-serializable.js @@ -0,0 +1,24 @@ +'use strict'; + +// Flags: --expose-internals + +// Check that main thread handles unserializable errors in a worker thread as +// expected. + +const common = require('../common'); + +const assert = require('assert'); + +const { Worker } = require('worker_threads'); + +const worker = new Worker(` + const { internalBinding } = require('internal/test/binding'); + const { getEnvMessagePort } = internalBinding('worker'); + const { messageTypes } = require('internal/worker/io'); + const messagePort = getEnvMessagePort(); + messagePort.postMessage({ type: messageTypes.COULD_NOT_SERIALIZE_ERROR }); +`, { eval: true }); + +worker.on('error', common.mustCall((e) => { + assert.strictEqual(e.code, 'ERR_WORKER_UNSERIALIZABLE_ERROR'); +})); diff --git a/test/parallel/test-worker-message-type-unknown.js b/test/parallel/test-worker-message-type-unknown.js new file mode 100644 index 00000000000000..32f6c2a32bd6b4 --- /dev/null +++ b/test/parallel/test-worker-message-type-unknown.js @@ -0,0 +1,25 @@ +'use strict'; + +// Check that main thread handles an unknown message type from a worker thread +// as expected. + +require('../common'); + +const assert = require('assert'); +const { spawnSync } = require('child_process'); + +const { Worker } = require('worker_threads'); +if (process.argv[2] !== 'spawned') { + const result = spawnSync(process.execPath, + [ '--expose-internals', __filename, 'spawned'], + { encoding: 'utf8' }); + assert.ok(result.stderr.includes('Unknown worker message type FHQWHGADS'), + `Expected error not found in: ${result.stderr}`); +} else { + new Worker(` + const { internalBinding } = require('internal/test/binding'); + const { getEnvMessagePort } = internalBinding('worker'); + const messagePort = getEnvMessagePort(); + messagePort.postMessage({ type: 'FHQWHGADS' }); + `, { eval: true }); +} diff --git a/test/sequential/test-heap-prof.js b/test/sequential/test-heap-prof.js new file mode 100644 index 00000000000000..cf70fa926091a6 --- /dev/null +++ b/test/sequential/test-heap-prof.js @@ -0,0 +1,375 @@ +'use strict'; + +// This tests that --heap-prof, --heap-prof-dir and --heap-prof-name works. + +const common = require('../common'); + +const fixtures = require('../common/fixtures'); +common.skipIfInspectorDisabled(); + +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const { spawnSync } = require('child_process'); + +const tmpdir = require('../common/tmpdir'); + +function getHeapProfiles(dir) { + const list = fs.readdirSync(dir); + return list + .filter((file) => file.endsWith('.heapprofile')) + .map((file) => path.join(dir, file)); +} + +function findFirstFrameInNode(root, func) { + const first = root.children.find( + (child) => child.callFrame.functionName === func + ); + if (first) { + return first; + } + for (const child of root.children) { + const first = findFirstFrameInNode(child, func); + if (first) { + return first; + } + } + return undefined; +} + +function findFirstFrame(file, func) { + const data = fs.readFileSync(file, 'utf8'); + const profile = JSON.parse(data); + const first = findFirstFrameInNode(profile.head, func); + return { frame: first, roots: profile.head.children }; +} + +function verifyFrames(output, file, func) { + const { frame, roots } = findFirstFrame(file, func); + if (!frame) { + // Show native debug output and the profile for debugging. + console.log(output.stderr.toString()); + console.log(roots); + } + assert.notDeepStrictEqual(frame, undefined); +} + +// We need to set --heap-prof-interval to a small enough value to make +// sure we can find our workload in the samples, so we need to set +// TEST_ALLOCATION > kHeapProfInterval. +const kHeapProfInterval = 128; +const TEST_ALLOCATION = kHeapProfInterval * 2; + +const env = { + ...process.env, + TEST_ALLOCATION, + NODE_DEBUG_NATIVE: 'INSPECTOR_PROFILER' +}; + +// Test --heap-prof without --heap-prof-interval. Here we just verify that +// we manage to generate a profile. +{ + tmpdir.refresh(); + const output = spawnSync(process.execPath, [ + '--heap-prof', + fixtures.path('workload', 'allocation.js'), + ], { + cwd: tmpdir.path, + env + }); + if (output.status !== 0) { + console.log(output.stderr.toString()); + console.log(output); + } + assert.strictEqual(output.status, 0); + const profiles = getHeapProfiles(tmpdir.path); + assert.strictEqual(profiles.length, 1); +} + +// Outputs heap profile when event loop is drained. +// TODO(joyeecheung): share the fixutres with v8 coverage tests +{ + tmpdir.refresh(); + const output = spawnSync(process.execPath, [ + '--heap-prof', + '--heap-prof-interval', + kHeapProfInterval, + fixtures.path('workload', 'allocation.js'), + ], { + cwd: tmpdir.path, + env + }); + if (output.status !== 0) { + console.log(output.stderr.toString()); + console.log(output); + } + assert.strictEqual(output.status, 0); + const profiles = getHeapProfiles(tmpdir.path); + assert.strictEqual(profiles.length, 1); + verifyFrames(output, profiles[0], 'runAllocation'); +} + +// Outputs heap profile when process.exit(55) exits process. +{ + tmpdir.refresh(); + const output = spawnSync(process.execPath, [ + '--heap-prof', + '--heap-prof-interval', + kHeapProfInterval, + fixtures.path('workload', 'allocation-exit.js'), + ], { + cwd: tmpdir.path, + env + }); + if (output.status !== 55) { + console.log(output.stderr.toString()); + } + assert.strictEqual(output.status, 55); + const profiles = getHeapProfiles(tmpdir.path); + assert.strictEqual(profiles.length, 1); + verifyFrames(output, profiles[0], 'runAllocation'); +} + +// Outputs heap profile when process.kill(process.pid, "SIGINT"); exits process. +{ + tmpdir.refresh(); + const output = spawnSync(process.execPath, [ + '--heap-prof', + '--heap-prof-interval', + kHeapProfInterval, + fixtures.path('workload', 'allocation-sigint.js'), + ], { + cwd: tmpdir.path, + env + }); + if (!common.isWindows) { + if (output.signal !== 'SIGINT') { + console.log(output.stderr.toString()); + } + assert.strictEqual(output.signal, 'SIGINT'); + } + const profiles = getHeapProfiles(tmpdir.path); + assert.strictEqual(profiles.length, 1); + verifyFrames(output, profiles[0], 'runAllocation'); +} + +// Outputs heap profile from worker when execArgv is set. +{ + tmpdir.refresh(); + const output = spawnSync(process.execPath, [ + fixtures.path('workload', 'allocation-worker-argv.js'), + ], { + cwd: tmpdir.path, + env: { + ...process.env, + HEAP_PROF_INTERVAL: '128' + } + }); + if (output.status !== 0) { + console.log(output.stderr.toString()); + } + assert.strictEqual(output.status, 0); + const profiles = getHeapProfiles(tmpdir.path); + assert.strictEqual(profiles.length, 1); + verifyFrames(output, profiles[0], 'runAllocation'); +} + +// --heap-prof-name without --heap-prof +{ + tmpdir.refresh(); + const output = spawnSync(process.execPath, [ + '--heap-prof-name', + 'test.heapprofile', + fixtures.path('workload', 'allocation.js'), + ], { + cwd: tmpdir.path, + env + }); + const stderr = output.stderr.toString().trim(); + if (output.status !== 9) { + console.log(stderr); + } + assert.strictEqual(output.status, 9); + assert.strictEqual( + stderr, + `${process.execPath}: --heap-prof-name must be used with --heap-prof`); +} + +// --heap-prof-dir without --heap-prof +{ + tmpdir.refresh(); + const output = spawnSync(process.execPath, [ + '--heap-prof-dir', + 'prof', + fixtures.path('workload', 'allocation.js'), + ], { + cwd: tmpdir.path, + env + }); + const stderr = output.stderr.toString().trim(); + if (output.status !== 9) { + console.log(stderr); + } + assert.strictEqual(output.status, 9); + assert.strictEqual( + stderr, + `${process.execPath}: --heap-prof-dir must be used with --heap-prof`); +} + +// --heap-prof-interval without --heap-prof +{ + tmpdir.refresh(); + const output = spawnSync(process.execPath, [ + '--heap-prof-interval', + kHeapProfInterval, + fixtures.path('workload', 'allocation.js'), + ], { + cwd: tmpdir.path, + env + }); + const stderr = output.stderr.toString().trim(); + if (output.status !== 9) { + console.log(stderr); + } + assert.strictEqual(output.status, 9); + assert.strictEqual( + stderr, + `${process.execPath}: ` + + '--heap-prof-interval must be used with --heap-prof'); +} + +// --heap-prof-name +{ + tmpdir.refresh(); + const file = path.join(tmpdir.path, 'test.heapprofile'); + const output = spawnSync(process.execPath, [ + '--heap-prof', + '--heap-prof-name', + 'test.heapprofile', + '--heap-prof-interval', + kHeapProfInterval, + fixtures.path('workload', 'allocation.js'), + ], { + cwd: tmpdir.path, + env + }); + if (output.status !== 0) { + console.log(output.stderr.toString()); + } + assert.strictEqual(output.status, 0); + const profiles = getHeapProfiles(tmpdir.path); + assert.deepStrictEqual(profiles, [file]); + verifyFrames(output, file, 'runAllocation'); +} + +// relative --heap-prof-dir +{ + tmpdir.refresh(); + const output = spawnSync(process.execPath, [ + '--heap-prof', + '--heap-prof-dir', + 'prof', + '--heap-prof-interval', + kHeapProfInterval, + fixtures.path('workload', 'allocation.js'), + ], { + cwd: tmpdir.path, + env + }); + if (output.status !== 0) { + console.log(output.stderr.toString()); + } + assert.strictEqual(output.status, 0); + const dir = path.join(tmpdir.path, 'prof'); + assert(fs.existsSync(dir)); + const profiles = getHeapProfiles(dir); + assert.strictEqual(profiles.length, 1); + verifyFrames(output, profiles[0], 'runAllocation'); +} + +// absolute --heap-prof-dir +{ + tmpdir.refresh(); + const dir = path.join(tmpdir.path, 'prof'); + const output = spawnSync(process.execPath, [ + '--heap-prof', + '--heap-prof-dir', + dir, + '--heap-prof-interval', + kHeapProfInterval, + fixtures.path('workload', 'allocation.js'), + ], { + cwd: tmpdir.path, + env + }); + if (output.status !== 0) { + console.log(output.stderr.toString()); + } + assert.strictEqual(output.status, 0); + assert(fs.existsSync(dir)); + const profiles = getHeapProfiles(dir); + assert.strictEqual(profiles.length, 1); + verifyFrames(output, profiles[0], 'runAllocation'); +} + +// --heap-prof-dir and --heap-prof-name +{ + tmpdir.refresh(); + const dir = path.join(tmpdir.path, 'prof'); + const file = path.join(dir, 'test.heapprofile'); + const output = spawnSync(process.execPath, [ + '--heap-prof', + '--heap-prof-name', + 'test.heapprofile', + '--heap-prof-dir', + dir, + '--heap-prof-interval', + kHeapProfInterval, + fixtures.path('workload', 'allocation.js'), + ], { + cwd: tmpdir.path, + env + }); + if (output.status !== 0) { + console.log(output.stderr.toString()); + } + assert.strictEqual(output.status, 0); + assert(fs.existsSync(dir)); + const profiles = getHeapProfiles(dir); + assert.deepStrictEqual(profiles, [file]); + verifyFrames(output, file, 'runAllocation'); +} + +{ + tmpdir.refresh(); + const output = spawnSync(process.execPath, [ + '--heap-prof-interval', + kHeapProfInterval, + '--heap-prof-dir', + 'prof', + '--heap-prof', + fixtures.path('workload', 'allocation-worker.js'), + ], { + cwd: tmpdir.path, + env + }); + if (output.status !== 0) { + console.log(output.stderr.toString()); + } + assert.strictEqual(output.status, 0); + const dir = path.join(tmpdir.path, 'prof'); + assert(fs.existsSync(dir)); + const profiles = getHeapProfiles(dir); + assert.strictEqual(profiles.length, 2); + const profile1 = findFirstFrame(profiles[0], 'runAllocation'); + const profile2 = findFirstFrame(profiles[1], 'runAllocation'); + if (!profile1.frame && !profile2.frame) { + // Show native debug output and the profile for debugging. + console.log(output.stderr.toString()); + console.log('heap path: ', profiles[0]); + console.log(profile1.roots); + console.log('heap path: ', profiles[1]); + console.log(profile2.roots); + } + assert(profile1.frame || profile2.frame); +} diff --git a/test/sequential/test-inspector-debug-end.js b/test/sequential/test-inspector-debug-end.js index d73e7dccc1a8fe..4c775981f1f3c9 100644 --- a/test/sequential/test-inspector-debug-end.js +++ b/test/sequential/test-inspector-debug-end.js @@ -10,14 +10,14 @@ async function testNoServerNoCrash() { const instance = new NodeInstance([], `process._debugEnd(); process.exit(42);`); - strictEqual(42, (await instance.expectShutdown()).exitCode); + strictEqual((await instance.expectShutdown()).exitCode, 42); } async function testNoSessionNoCrash() { console.log('Test there\'s no crash stopping server without connecting'); const instance = new NodeInstance('--inspect=0', 'process._debugEnd();process.exit(42);'); - strictEqual(42, (await instance.expectShutdown()).exitCode); + strictEqual((await instance.expectShutdown()).exitCode, 42); } async function testSessionNoCrash() { @@ -33,7 +33,7 @@ async function testSessionNoCrash() { const session = await instance.connectInspectorSession(); await session.send({ 'method': 'Runtime.runIfWaitingForDebugger' }); await session.waitForServerDisconnect(); - strictEqual(42, (await instance.expectShutdown()).exitCode); + strictEqual((await instance.expectShutdown()).exitCode, 42); } async function runTest() { diff --git a/test/sequential/test-performance.js b/test/sequential/test-perf-hooks.js similarity index 100% rename from test/sequential/test-performance.js rename to test/sequential/test-perf-hooks.js diff --git a/test/v8-updates/test-postmortem-metadata.js b/test/v8-updates/test-postmortem-metadata.js index c6704ef1e34bc2..3dacc97388993d 100644 --- a/test/v8-updates/test-postmortem-metadata.js +++ b/test/v8-updates/test-postmortem-metadata.js @@ -41,6 +41,9 @@ const symbols = nm.stdout.toString().split('\n').reduce((filtered, line) => { return filtered; }, []); + +assert.notStrictEqual(symbols.length, 0, 'No postmortem metadata detected'); + const missing = getExpectedSymbols().filter((symbol) => { return !symbols.includes(symbol); }); diff --git a/test/wpt/status/encoding.json b/test/wpt/status/encoding.json index 5cf387e554844e..a81ba605c1e72f 100644 --- a/test/wpt/status/encoding.json +++ b/test/wpt/status/encoding.json @@ -47,5 +47,11 @@ }, "unsupported-encodings.any.js": { "skip": "decoding-helpers.js needs XMLHttpRequest" + }, + "streams/*.js": { + "fail": "No implementation of TextDecoderStream and TextEncoderStream" + }, + "encodeInto.any.js": { + "fail": "TextEncoder.prototype.encodeInto not implemented" } } diff --git a/tools/doc/html.js b/tools/doc/html.js index efdc8b0d475b0f..318feefe3461a1 100644 --- a/tools/doc/html.js +++ b/tools/doc/html.js @@ -23,6 +23,7 @@ const common = require('./common.js'); const fs = require('fs'); +const getVersions = require('./versions.js'); const unified = require('unified'); const find = require('unist-util-find'); const visit = require('unist-util-visit'); @@ -62,7 +63,7 @@ const gtocHTML = unified() const templatePath = path.join(docPath, 'template.html'); const template = fs.readFileSync(templatePath, 'utf8'); -function toHTML({ input, content, filename, nodeVersion }, cb) { +async function toHTML({ input, content, filename, nodeVersion }, cb) { filename = path.basename(filename, '.md'); const id = filename.replace(/\W+/g, '-'); @@ -80,7 +81,7 @@ function toHTML({ input, content, filename, nodeVersion }, cb) { const docCreated = input.match( //); if (docCreated) { - HTML = HTML.replace('__ALTDOCS__', altDocs(filename, docCreated)); + HTML = HTML.replace('__ALTDOCS__', await altDocs(filename, docCreated)); } else { console.error(`Failed to add alternative version links to ${filename}`); HTML = HTML.replace('__ALTDOCS__', ''); @@ -390,22 +391,10 @@ function getId(text, idCounters) { return text; } -function altDocs(filename, docCreated) { +async function altDocs(filename, docCreated) { const [, docCreatedMajor, docCreatedMinor] = docCreated.map(Number); const host = 'https://nodejs.org'; - const versions = [ - { num: '12.x' }, - { num: '11.x' }, - { num: '10.x', lts: true }, - { num: '9.x' }, - { num: '8.x', lts: true }, - { num: '7.x' }, - { num: '6.x' }, - { num: '5.x' }, - { num: '4.x' }, - { num: '0.12.x' }, - { num: '0.10.x' }, - ]; + const versions = await getVersions.versions(); const getHref = (versionNum) => `${host}/docs/latest-v${versionNum}/api/${filename}.html`; diff --git a/tools/doc/package-lock.json b/tools/doc/package-lock.json index 7a2bd4991a8690..88aa99f61835d3 100644 --- a/tools/doc/package-lock.json +++ b/tools/doc/package-lock.json @@ -321,9 +321,9 @@ "integrity": "sha512-T3FlsX8rCHAH8e7RE7PfOPZVFQlcV3XRF9eOOBQ1uf70OxO7CjjSOjeImMPCADBdYWcStAbVbYvJ1m2D3tb+EA==" }, "js-yaml": { - "version": "3.12.2", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.12.2.tgz", - "integrity": "sha512-QHn/Lh/7HhZ/Twc7vJYQTkjuCa0kaCcDcjK5Zlk2rvnUpy7DxMJ23+Jc2dcyvltwQVg1nygAVlB2oRDFHoRS5Q==", + "version": "3.13.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz", + "integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==", "dev": true, "requires": { "argparse": "^1.0.7", diff --git a/tools/doc/package.json b/tools/doc/package.json index 89648a84903e5f..1d2953b023b06d 100644 --- a/tools/doc/package.json +++ b/tools/doc/package.json @@ -19,7 +19,7 @@ "unist-util-visit": "^1.4.0" }, "devDependencies": { - "js-yaml": "^3.12.2" + "js-yaml": "^3.13.1" }, "optionalDependencies": {}, "bin": "./generate.js" diff --git a/tools/doc/versions.js b/tools/doc/versions.js new file mode 100644 index 00000000000000..854329bd9a8a02 --- /dev/null +++ b/tools/doc/versions.js @@ -0,0 +1,45 @@ +'use strict'; + +let _versions; + +const getUrl = (url) => { + return new Promise((resolve, reject) => { + const https = require('https'); + const request = https.get(url, (response) => { + if (response.statusCode !== 200) { + reject(new Error( + `Failed to get ${url}, status code ${response.statusCode}`)); + } + response.setEncoding('utf8'); + let body = ''; + response.on('data', (data) => body += data); + response.on('end', () => resolve(body)); + }); + request.on('error', (err) => reject(err)); + }); +}; + +module.exports = { + async versions() { + if (_versions) { + return _versions; + } + + // The CHANGELOG.md on release branches may not reference newer semver + // majors of Node.js so fetch and parse the version from the master branch. + const githubContentUrl = 'https://raw.githubusercontent.com/nodejs/node/'; + const changelog = await getUrl(`${githubContentUrl}/master/CHANGELOG.md`); + const ltsRE = /Long Term Support/i; + const versionRE = /\* \[Node\.js ([0-9.]+)\][^-—]+[-—]\s*(.*)\n/g; + _versions = []; + let match; + while ((match = versionRE.exec(changelog)) != null) { + const entry = { num: `${match[1]}.x` }; + if (ltsRE.test(match[2])) { + entry.lts = true; + } + _versions.push(entry); + } + return _versions; + } +}; diff --git a/tools/inspector_protocol/.clang-format b/tools/inspector_protocol/.clang-format new file mode 100644 index 00000000000000..fcbc9c321a5c61 --- /dev/null +++ b/tools/inspector_protocol/.clang-format @@ -0,0 +1,36 @@ +# Defines the Chromium style for automatic reformatting. +# http://clang.llvm.org/docs/ClangFormatStyleOptions.html +BasedOnStyle: Chromium +# This defaults to 'Auto'. Explicitly set it for a while, so that +# 'vector >' in existing files gets formatted to +# 'vector>'. ('Auto' means that clang-format will only use +# 'int>>' if the file already contains at least one such instance.) +Standard: Cpp11 + +# Make sure code like: +# IPC_BEGIN_MESSAGE_MAP() +# IPC_MESSAGE_HANDLER(WidgetHostViewHost_Update, OnUpdate) +# IPC_END_MESSAGE_MAP() +# gets correctly indented. +MacroBlockBegin: "^\ +BEGIN_MSG_MAP|\ +BEGIN_MSG_MAP_EX|\ +BEGIN_SAFE_MSG_MAP_EX|\ +CR_BEGIN_MSG_MAP_EX|\ +IPC_BEGIN_MESSAGE_MAP|\ +IPC_BEGIN_MESSAGE_MAP_WITH_PARAM|\ +IPC_PROTOBUF_MESSAGE_TRAITS_BEGIN|\ +IPC_STRUCT_BEGIN|\ +IPC_STRUCT_BEGIN_WITH_PARENT|\ +IPC_STRUCT_TRAITS_BEGIN|\ +POLPARAMS_BEGIN|\ +PPAPI_BEGIN_MESSAGE_MAP$" +MacroBlockEnd: "^\ +CR_END_MSG_MAP|\ +END_MSG_MAP|\ +IPC_END_MESSAGE_MAP|\ +IPC_PROTOBUF_MESSAGE_TRAITS_END|\ +IPC_STRUCT_END|\ +IPC_STRUCT_TRAITS_END|\ +POLPARAMS_END|\ +PPAPI_END_MESSAGE_MAP$" diff --git a/tools/inspector_protocol/BUILD.gn b/tools/inspector_protocol/BUILD.gn new file mode 100644 index 00000000000000..974471bf2718d5 --- /dev/null +++ b/tools/inspector_protocol/BUILD.gn @@ -0,0 +1,34 @@ +# Copyright 2019 the V8 project authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +static_library("encoding") { + sources = [ + "encoding/encoding.cc", + "encoding/encoding.h", + ] +} + +# encoding_test is part of the unittests, defined in +# test/unittests/BUILD.gn. + +import("../../gni/v8.gni") + +v8_source_set("encoding_test") { + sources = [ + "encoding/encoding_test.cc", + "encoding/encoding_test_helper.h", + ] + configs = [ + "../..:external_config", + "../..:internal_config_base", + ] + deps = [ + ":encoding", + "../..:v8_libbase", + "../../src/inspector:inspector_string_conversions", + "//testing/gmock", + "//testing/gtest", + ] + testonly = true +} diff --git a/tools/inspector_protocol/README.md b/tools/inspector_protocol/README.md new file mode 100644 index 00000000000000..da3f93f3f3b49e --- /dev/null +++ b/tools/inspector_protocol/README.md @@ -0,0 +1,33 @@ +# Chromium inspector (devtools) protocol + +This package contains code generators and templates for the Chromium +inspector protocol. + +The canonical location of this package is at +https://chromium.googlesource.com/deps/inspector_protocol/ + +In the Chromium tree, it's rolled into +https://cs.chromium.org/chromium/src/third_party/inspector_protocol/ + +In the V8 tree, it's rolled into +https://cs.chromium.org/chromium/src/v8/third_party/inspector_protocol/ + +See also [Contributing to Chrome Devtools Protocol](https://docs.google.com/document/d/1c-COD2kaK__5iMM5SEx-PzNA7HFmgttcYfOHHX0HaOM/edit). + +We're working on enabling standalone builds for parts of this package for +testing and development, please feel free to ignore this for now. +But, if you're familiar with +[Chromium's development process](https://www.chromium.org/developers/contributing-code) +and have the depot_tools installed, you may use these commands +to fetch the package (and dependencies) and build and run the tests: + + fetch inspector_protocol + cd src + gn gen out/Release + ninja -C out/Release json_parser_test + out/Release/json_parser_test + +You'll probably also need to install g++, since Clang uses this to find the +standard C++ headers. E.g., + + sudo apt-get install g++-8 diff --git a/tools/inspector_protocol/README.node b/tools/inspector_protocol/README.node index 6f22020a4de3dc..a8380198576d46 100644 --- a/tools/inspector_protocol/README.node +++ b/tools/inspector_protocol/README.node @@ -2,7 +2,7 @@ Name: inspector protocol Short Name: inspector_protocol URL: https://chromium.googlesource.com/deps/inspector_protocol/ Version: 0 -Revision: f67ec5180f476830e839226b5ca948e43070fdab +Revision: 0aafd2876f7485db7b07c513c0457b7cbbbe3304 License: BSD License File: LICENSE Security Critical: no diff --git a/tools/inspector_protocol/code_generator.py b/tools/inspector_protocol/code_generator.py index 9200022413303a..7b555d7478a0c7 100755 --- a/tools/inspector_protocol/code_generator.py +++ b/tools/inspector_protocol/code_generator.py @@ -5,7 +5,7 @@ import os.path import sys -import optparse +import argparse import collections import functools import re @@ -17,6 +17,13 @@ import pdl +try: + unicode +except NameError: + # Define unicode for Py3 + def unicode(s, *_): + return s + # Path handling for libraries and templates # Paths have to be normalized because Jinja uses the exact template path to # determine the hash used in the cache filename, and we need a pre-caching step @@ -53,27 +60,16 @@ def init_defaults(config_tuple, path, defaults): return collections.namedtuple('X', keys)(*values) try: - cmdline_parser = optparse.OptionParser() - cmdline_parser.add_option("--output_base") - cmdline_parser.add_option("--jinja_dir") - cmdline_parser.add_option("--config") - cmdline_parser.add_option("--config_value", action="append", type="string") - arg_options, _ = cmdline_parser.parse_args() + cmdline_parser = argparse.ArgumentParser() + cmdline_parser.add_argument("--output_base", type=unicode, required=True) + cmdline_parser.add_argument("--jinja_dir", type=unicode, required=True) + cmdline_parser.add_argument("--config", type=unicode, required=True) + cmdline_parser.add_argument("--config_value", default=[], action="append") + arg_options = cmdline_parser.parse_args() jinja_dir = arg_options.jinja_dir - if not jinja_dir: - raise Exception("jinja directory must be specified") - jinja_dir = jinja_dir.decode('utf8') output_base = arg_options.output_base - if not output_base: - raise Exception("Base output directory must be specified") - output_base = output_base.decode('utf8') config_file = arg_options.config - if not config_file: - raise Exception("Config file name must be specified") - config_file = config_file.decode('utf8') config_values = arg_options.config_value - if not config_values: - config_values = [] except Exception: # Work with python 2 and 3 http://docs.python.org/py3k/howto/pyporting.html exc = sys.exc_info()[1] @@ -631,7 +627,7 @@ def main(): "Array_h.template", "DispatcherBase_h.template", "Parser_h.template", - "CBOR_h.template", + "encoding_h.template", ] protocol_cpp_templates = [ @@ -641,7 +637,7 @@ def main(): "Object_cpp.template", "DispatcherBase_cpp.template", "Parser_cpp.template", - "CBOR_cpp.template", + "encoding_cpp.template", ] forward_h_templates = [ diff --git a/tools/inspector_protocol/codereview.settings b/tools/inspector_protocol/codereview.settings new file mode 100644 index 00000000000000..6ac8580b4ccd9f --- /dev/null +++ b/tools/inspector_protocol/codereview.settings @@ -0,0 +1,6 @@ +# This file is used by git-cl to get repository specific information. +CC_LIST: chromium-reviews@chromium.org +CODE_REVIEW_SERVER: codereview.chromium.org +GERRIT_HOST: True +PROJECT: inspector_protocol +VIEW_VC: https://chromium.googlesource.com/deps/inspector_protocol/+/ diff --git a/tools/inspector_protocol/convert_protocol_to_json.py b/tools/inspector_protocol/convert_protocol_to_json.py index 96048f793d85a8..f98bebcd5e66c5 100755 --- a/tools/inspector_protocol/convert_protocol_to_json.py +++ b/tools/inspector_protocol/convert_protocol_to_json.py @@ -4,10 +4,8 @@ # found in the LICENSE file. import argparse -import collections import json import os.path -import re import sys import pdl diff --git a/tools/inspector_protocol/encoding/encoding.cc b/tools/inspector_protocol/encoding/encoding.cc new file mode 100644 index 00000000000000..f7858c9a22ba64 --- /dev/null +++ b/tools/inspector_protocol/encoding/encoding.cc @@ -0,0 +1,2189 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "encoding.h" + +#include +#include +#include +#include +#include +#include + +namespace v8_inspector_protocol_encoding { +// ============================================================================= +// Status and Error codes +// ============================================================================= + +std::string Status::ToASCIIString() const { + switch (error) { + case Error::OK: + return "OK"; + case Error::JSON_PARSER_UNPROCESSED_INPUT_REMAINS: + return ToASCIIString("JSON: unprocessed input remains"); + case Error::JSON_PARSER_STACK_LIMIT_EXCEEDED: + return ToASCIIString("JSON: stack limit exceeded"); + case Error::JSON_PARSER_NO_INPUT: + return ToASCIIString("JSON: no input"); + case Error::JSON_PARSER_INVALID_TOKEN: + return ToASCIIString("JSON: invalid token"); + case Error::JSON_PARSER_INVALID_NUMBER: + return ToASCIIString("JSON: invalid number"); + case Error::JSON_PARSER_INVALID_STRING: + return ToASCIIString("JSON: invalid string"); + case Error::JSON_PARSER_UNEXPECTED_ARRAY_END: + return ToASCIIString("JSON: unexpected array end"); + case Error::JSON_PARSER_COMMA_OR_ARRAY_END_EXPECTED: + return ToASCIIString("JSON: comma or array end expected"); + case Error::JSON_PARSER_STRING_LITERAL_EXPECTED: + return ToASCIIString("JSON: string literal expected"); + case Error::JSON_PARSER_COLON_EXPECTED: + return ToASCIIString("JSON: colon expected"); + case Error::JSON_PARSER_UNEXPECTED_MAP_END: + return ToASCIIString("JSON: unexpected map end"); + case Error::JSON_PARSER_COMMA_OR_MAP_END_EXPECTED: + return ToASCIIString("JSON: comma or map end expected"); + case Error::JSON_PARSER_VALUE_EXPECTED: + return ToASCIIString("JSON: value expected"); + + case Error::CBOR_INVALID_INT32: + return ToASCIIString("CBOR: invalid int32"); + case Error::CBOR_INVALID_DOUBLE: + return ToASCIIString("CBOR: invalid double"); + case Error::CBOR_INVALID_ENVELOPE: + return ToASCIIString("CBOR: invalid envelope"); + case Error::CBOR_INVALID_STRING8: + return ToASCIIString("CBOR: invalid string8"); + case Error::CBOR_INVALID_STRING16: + return ToASCIIString("CBOR: invalid string16"); + case Error::CBOR_INVALID_BINARY: + return ToASCIIString("CBOR: invalid binary"); + case Error::CBOR_UNSUPPORTED_VALUE: + return ToASCIIString("CBOR: unsupported value"); + case Error::CBOR_NO_INPUT: + return ToASCIIString("CBOR: no input"); + case Error::CBOR_INVALID_START_BYTE: + return ToASCIIString("CBOR: invalid start byte"); + case Error::CBOR_UNEXPECTED_EOF_EXPECTED_VALUE: + return ToASCIIString("CBOR: unexpected eof expected value"); + case Error::CBOR_UNEXPECTED_EOF_IN_ARRAY: + return ToASCIIString("CBOR: unexpected eof in array"); + case Error::CBOR_UNEXPECTED_EOF_IN_MAP: + return ToASCIIString("CBOR: unexpected eof in map"); + case Error::CBOR_INVALID_MAP_KEY: + return ToASCIIString("CBOR: invalid map key"); + case Error::CBOR_STACK_LIMIT_EXCEEDED: + return ToASCIIString("CBOR: stack limit exceeded"); + case Error::CBOR_TRAILING_JUNK: + return ToASCIIString("CBOR: trailing junk"); + case Error::CBOR_MAP_START_EXPECTED: + return ToASCIIString("CBOR: map start expected"); + case Error::CBOR_MAP_STOP_EXPECTED: + return ToASCIIString("CBOR: map stop expected"); + case Error::CBOR_ENVELOPE_SIZE_LIMIT_EXCEEDED: + return ToASCIIString("CBOR: envelope size limit exceeded"); + } + // Some compilers can't figure out that we can't get here. + return "INVALID ERROR CODE"; +} + +std::string Status::ToASCIIString(const char* msg) const { + return std::string(msg) + " at position " + std::to_string(pos); +} + +namespace cbor { +namespace { +// Indicates the number of bits the "initial byte" needs to be shifted to the +// right after applying |kMajorTypeMask| to produce the major type in the +// lowermost bits. +static constexpr uint8_t kMajorTypeBitShift = 5u; +// Mask selecting the low-order 5 bits of the "initial byte", which is where +// the additional information is encoded. +static constexpr uint8_t kAdditionalInformationMask = 0x1f; +// Mask selecting the high-order 3 bits of the "initial byte", which indicates +// the major type of the encoded value. +static constexpr uint8_t kMajorTypeMask = 0xe0; +// Indicates the integer is in the following byte. +static constexpr uint8_t kAdditionalInformation1Byte = 24u; +// Indicates the integer is in the next 2 bytes. +static constexpr uint8_t kAdditionalInformation2Bytes = 25u; +// Indicates the integer is in the next 4 bytes. +static constexpr uint8_t kAdditionalInformation4Bytes = 26u; +// Indicates the integer is in the next 8 bytes. +static constexpr uint8_t kAdditionalInformation8Bytes = 27u; + +// Encodes the initial byte, consisting of the |type| in the first 3 bits +// followed by 5 bits of |additional_info|. +constexpr uint8_t EncodeInitialByte(MajorType type, uint8_t additional_info) { + return (static_cast(type) << kMajorTypeBitShift) | + (additional_info & kAdditionalInformationMask); +} + +// TAG 24 indicates that what follows is a byte string which is +// encoded in CBOR format. We use this as a wrapper for +// maps and arrays, allowing us to skip them, because the +// byte string carries its size (byte length). +// https://tools.ietf.org/html/rfc7049#section-2.4.4.1 +static constexpr uint8_t kInitialByteForEnvelope = + EncodeInitialByte(MajorType::TAG, 24); +// The initial byte for a byte string with at most 2^32 bytes +// of payload. This is used for envelope encoding, even if +// the byte string is shorter. +static constexpr uint8_t kInitialByteFor32BitLengthByteString = + EncodeInitialByte(MajorType::BYTE_STRING, 26); + +// See RFC 7049 Section 2.2.1, indefinite length arrays / maps have additional +// info = 31. +static constexpr uint8_t kInitialByteIndefiniteLengthArray = + EncodeInitialByte(MajorType::ARRAY, 31); +static constexpr uint8_t kInitialByteIndefiniteLengthMap = + EncodeInitialByte(MajorType::MAP, 31); +// See RFC 7049 Section 2.3, Table 1; this is used for finishing indefinite +// length maps / arrays. +static constexpr uint8_t kStopByte = + EncodeInitialByte(MajorType::SIMPLE_VALUE, 31); + +// See RFC 7049 Section 2.3, Table 2. +static constexpr uint8_t kEncodedTrue = + EncodeInitialByte(MajorType::SIMPLE_VALUE, 21); +static constexpr uint8_t kEncodedFalse = + EncodeInitialByte(MajorType::SIMPLE_VALUE, 20); +static constexpr uint8_t kEncodedNull = + EncodeInitialByte(MajorType::SIMPLE_VALUE, 22); +static constexpr uint8_t kInitialByteForDouble = + EncodeInitialByte(MajorType::SIMPLE_VALUE, 27); + +// See RFC 7049 Table 3 and Section 2.4.4.2. This is used as a prefix for +// arbitrary binary data encoded as BYTE_STRING. +static constexpr uint8_t kExpectedConversionToBase64Tag = + EncodeInitialByte(MajorType::TAG, 22); + +// Writes the bytes for |v| to |out|, starting with the most significant byte. +// See also: https://commandcenter.blogspot.com/2012/04/byte-order-fallacy.html +template +void WriteBytesMostSignificantByteFirst(T v, C* out) { + for (int shift_bytes = sizeof(T) - 1; shift_bytes >= 0; --shift_bytes) + out->push_back(0xff & (v >> (shift_bytes * 8))); +} + +// Extracts sizeof(T) bytes from |in| to extract a value of type T +// (e.g. uint64_t, uint32_t, ...), most significant byte first. +// See also: https://commandcenter.blogspot.com/2012/04/byte-order-fallacy.html +template +T ReadBytesMostSignificantByteFirst(span in) { + assert(in.size() >= sizeof(T)); + T result = 0; + for (size_t shift_bytes = 0; shift_bytes < sizeof(T); ++shift_bytes) + result |= T(in[sizeof(T) - 1 - shift_bytes]) << (shift_bytes * 8); + return result; +} +} // namespace + +namespace internals { +// Reads the start of a token with definitive size from |bytes|. +// |type| is the major type as specified in RFC 7049 Section 2.1. +// |value| is the payload (e.g. for MajorType::UNSIGNED) or is the size +// (e.g. for BYTE_STRING). +// If successful, returns the number of bytes read. Otherwise returns -1. +// TODO(johannes): change return type to size_t and use 0 for error. +int8_t ReadTokenStart(span bytes, MajorType* type, uint64_t* value) { + if (bytes.empty()) + return -1; + uint8_t initial_byte = bytes[0]; + *type = MajorType((initial_byte & kMajorTypeMask) >> kMajorTypeBitShift); + + uint8_t additional_information = initial_byte & kAdditionalInformationMask; + if (additional_information < 24) { + // Values 0-23 are encoded directly into the additional info of the + // initial byte. + *value = additional_information; + return 1; + } + if (additional_information == kAdditionalInformation1Byte) { + // Values 24-255 are encoded with one initial byte, followed by the value. + if (bytes.size() < 2) + return -1; + *value = ReadBytesMostSignificantByteFirst(bytes.subspan(1)); + return 2; + } + if (additional_information == kAdditionalInformation2Bytes) { + // Values 256-65535: 1 initial byte + 2 bytes payload. + if (bytes.size() < 1 + sizeof(uint16_t)) + return -1; + *value = ReadBytesMostSignificantByteFirst(bytes.subspan(1)); + return 3; + } + if (additional_information == kAdditionalInformation4Bytes) { + // 32 bit uint: 1 initial byte + 4 bytes payload. + if (bytes.size() < 1 + sizeof(uint32_t)) + return -1; + *value = ReadBytesMostSignificantByteFirst(bytes.subspan(1)); + return 5; + } + if (additional_information == kAdditionalInformation8Bytes) { + // 64 bit uint: 1 initial byte + 8 bytes payload. + if (bytes.size() < 1 + sizeof(uint64_t)) + return -1; + *value = ReadBytesMostSignificantByteFirst(bytes.subspan(1)); + return 9; + } + return -1; +} + +// Writes the start of a token with |type|. The |value| may indicate the size, +// or it may be the payload if the value is an unsigned integer. +template +void WriteTokenStartTmpl(MajorType type, uint64_t value, C* encoded) { + if (value < 24) { + // Values 0-23 are encoded directly into the additional info of the + // initial byte. + encoded->push_back(EncodeInitialByte(type, /*additional_info=*/value)); + return; + } + if (value <= std::numeric_limits::max()) { + // Values 24-255 are encoded with one initial byte, followed by the value. + encoded->push_back(EncodeInitialByte(type, kAdditionalInformation1Byte)); + encoded->push_back(value); + return; + } + if (value <= std::numeric_limits::max()) { + // Values 256-65535: 1 initial byte + 2 bytes payload. + encoded->push_back(EncodeInitialByte(type, kAdditionalInformation2Bytes)); + WriteBytesMostSignificantByteFirst(value, encoded); + return; + } + if (value <= std::numeric_limits::max()) { + // 32 bit uint: 1 initial byte + 4 bytes payload. + encoded->push_back(EncodeInitialByte(type, kAdditionalInformation4Bytes)); + WriteBytesMostSignificantByteFirst(static_cast(value), + encoded); + return; + } + // 64 bit uint: 1 initial byte + 8 bytes payload. + encoded->push_back(EncodeInitialByte(type, kAdditionalInformation8Bytes)); + WriteBytesMostSignificantByteFirst(value, encoded); +} +void WriteTokenStart(MajorType type, + uint64_t value, + std::vector* encoded) { + WriteTokenStartTmpl(type, value, encoded); +} +void WriteTokenStart(MajorType type, uint64_t value, std::string* encoded) { + WriteTokenStartTmpl(type, value, encoded); +} +} // namespace internals + +// ============================================================================= +// Detecting CBOR content +// ============================================================================= + +uint8_t InitialByteForEnvelope() { + return kInitialByteForEnvelope; +} +uint8_t InitialByteFor32BitLengthByteString() { + return kInitialByteFor32BitLengthByteString; +} +bool IsCBORMessage(span msg) { + return msg.size() >= 6 && msg[0] == InitialByteForEnvelope() && + msg[1] == InitialByteFor32BitLengthByteString(); +} + +// ============================================================================= +// Encoding invidiual CBOR items +// ============================================================================= + +uint8_t EncodeTrue() { + return kEncodedTrue; +} +uint8_t EncodeFalse() { + return kEncodedFalse; +} +uint8_t EncodeNull() { + return kEncodedNull; +} + +uint8_t EncodeIndefiniteLengthArrayStart() { + return kInitialByteIndefiniteLengthArray; +} + +uint8_t EncodeIndefiniteLengthMapStart() { + return kInitialByteIndefiniteLengthMap; +} + +uint8_t EncodeStop() { + return kStopByte; +} + +template +void EncodeInt32Tmpl(int32_t value, C* out) { + if (value >= 0) { + internals::WriteTokenStart(MajorType::UNSIGNED, value, out); + } else { + uint64_t representation = static_cast(-(value + 1)); + internals::WriteTokenStart(MajorType::NEGATIVE, representation, out); + } +} +void EncodeInt32(int32_t value, std::vector* out) { + EncodeInt32Tmpl(value, out); +} +void EncodeInt32(int32_t value, std::string* out) { + EncodeInt32Tmpl(value, out); +} + +template +void EncodeString16Tmpl(span in, C* out) { + uint64_t byte_length = static_cast(in.size_bytes()); + internals::WriteTokenStart(MajorType::BYTE_STRING, byte_length, out); + // When emitting UTF16 characters, we always write the least significant byte + // first; this is because it's the native representation for X86. + // TODO(johannes): Implement a more efficient thing here later, e.g. + // casting *iff* the machine has this byte order. + // The wire format for UTF16 chars will probably remain the same + // (least significant byte first) since this way we can have + // golden files, unittests, etc. that port easily and universally. + // See also: + // https://commandcenter.blogspot.com/2012/04/byte-order-fallacy.html + for (const uint16_t two_bytes : in) { + out->push_back(two_bytes); + out->push_back(two_bytes >> 8); + } +} +void EncodeString16(span in, std::vector* out) { + EncodeString16Tmpl(in, out); +} +void EncodeString16(span in, std::string* out) { + EncodeString16Tmpl(in, out); +} + +template +void EncodeString8Tmpl(span in, C* out) { + internals::WriteTokenStart(MajorType::STRING, + static_cast(in.size_bytes()), out); + out->insert(out->end(), in.begin(), in.end()); +} +void EncodeString8(span in, std::vector* out) { + EncodeString8Tmpl(in, out); +} +void EncodeString8(span in, std::string* out) { + EncodeString8Tmpl(in, out); +} + +template +void EncodeFromLatin1Tmpl(span latin1, C* out) { + for (size_t ii = 0; ii < latin1.size(); ++ii) { + if (latin1[ii] <= 127) + continue; + // If there's at least one non-ASCII char, convert to UTF8. + std::vector utf8(latin1.begin(), latin1.begin() + ii); + for (; ii < latin1.size(); ++ii) { + if (latin1[ii] <= 127) { + utf8.push_back(latin1[ii]); + } else { + // 0xC0 means it's a UTF8 sequence with 2 bytes. + utf8.push_back((latin1[ii] >> 6) | 0xc0); + utf8.push_back((latin1[ii] | 0x80) & 0xbf); + } + } + EncodeString8(SpanFrom(utf8), out); + return; + } + EncodeString8(latin1, out); +} +void EncodeFromLatin1(span latin1, std::vector* out) { + EncodeFromLatin1Tmpl(latin1, out); +} +void EncodeFromLatin1(span latin1, std::string* out) { + EncodeFromLatin1Tmpl(latin1, out); +} + +template +void EncodeFromUTF16Tmpl(span utf16, C* out) { + // If there's at least one non-ASCII char, encode as STRING16 (UTF16). + for (uint16_t ch : utf16) { + if (ch <= 127) + continue; + EncodeString16(utf16, out); + return; + } + // It's all US-ASCII, strip out every second byte and encode as UTF8. + internals::WriteTokenStart(MajorType::STRING, + static_cast(utf16.size()), out); + out->insert(out->end(), utf16.begin(), utf16.end()); +} +void EncodeFromUTF16(span utf16, std::vector* out) { + EncodeFromUTF16Tmpl(utf16, out); +} +void EncodeFromUTF16(span utf16, std::string* out) { + EncodeFromUTF16Tmpl(utf16, out); +} + +template +void EncodeBinaryTmpl(span in, C* out) { + out->push_back(kExpectedConversionToBase64Tag); + uint64_t byte_length = static_cast(in.size_bytes()); + internals::WriteTokenStart(MajorType::BYTE_STRING, byte_length, out); + out->insert(out->end(), in.begin(), in.end()); +} +void EncodeBinary(span in, std::vector* out) { + EncodeBinaryTmpl(in, out); +} +void EncodeBinary(span in, std::string* out) { + EncodeBinaryTmpl(in, out); +} + +// A double is encoded with a specific initial byte +// (kInitialByteForDouble) plus the 64 bits of payload for its value. +constexpr size_t kEncodedDoubleSize = 1 + sizeof(uint64_t); + +// An envelope is encoded with a specific initial byte +// (kInitialByteForEnvelope), plus the start byte for a BYTE_STRING with a 32 +// bit wide length, plus a 32 bit length for that string. +constexpr size_t kEncodedEnvelopeHeaderSize = 1 + 1 + sizeof(uint32_t); + +template +void EncodeDoubleTmpl(double value, C* out) { + // The additional_info=27 indicates 64 bits for the double follow. + // See RFC 7049 Section 2.3, Table 1. + out->push_back(kInitialByteForDouble); + union { + double from_double; + uint64_t to_uint64; + } reinterpret; + reinterpret.from_double = value; + WriteBytesMostSignificantByteFirst(reinterpret.to_uint64, out); +} +void EncodeDouble(double value, std::vector* out) { + EncodeDoubleTmpl(value, out); +} +void EncodeDouble(double value, std::string* out) { + EncodeDoubleTmpl(value, out); +} + +// ============================================================================= +// cbor::EnvelopeEncoder - for wrapping submessages +// ============================================================================= + +template +void EncodeStartTmpl(C* out, size_t* byte_size_pos) { + assert(*byte_size_pos == 0); + out->push_back(kInitialByteForEnvelope); + out->push_back(kInitialByteFor32BitLengthByteString); + *byte_size_pos = out->size(); + out->resize(out->size() + sizeof(uint32_t)); +} + +void EnvelopeEncoder::EncodeStart(std::vector* out) { + EncodeStartTmpl>(out, &byte_size_pos_); +} + +void EnvelopeEncoder::EncodeStart(std::string* out) { + EncodeStartTmpl(out, &byte_size_pos_); +} + +template +bool EncodeStopTmpl(C* out, size_t* byte_size_pos) { + assert(*byte_size_pos != 0); + // The byte size is the size of the payload, that is, all the + // bytes that were written past the byte size position itself. + uint64_t byte_size = out->size() - (*byte_size_pos + sizeof(uint32_t)); + // We store exactly 4 bytes, so at most INT32MAX, with most significant + // byte first. + if (byte_size > std::numeric_limits::max()) + return false; + for (int shift_bytes = sizeof(uint32_t) - 1; shift_bytes >= 0; + --shift_bytes) { + (*out)[(*byte_size_pos)++] = 0xff & (byte_size >> (shift_bytes * 8)); + } + return true; +} + +bool EnvelopeEncoder::EncodeStop(std::vector* out) { + return EncodeStopTmpl(out, &byte_size_pos_); +} + +bool EnvelopeEncoder::EncodeStop(std::string* out) { + return EncodeStopTmpl(out, &byte_size_pos_); +} + +// ============================================================================= +// cbor::NewCBOREncoder - for encoding from a streaming parser +// ============================================================================= + +namespace { +template +class CBOREncoder : public StreamingParserHandler { + public: + CBOREncoder(C* out, Status* status) : out_(out), status_(status) { + *status_ = Status(); + } + + void HandleMapBegin() override { + if (!status_->ok()) + return; + envelopes_.emplace_back(); + envelopes_.back().EncodeStart(out_); + out_->push_back(kInitialByteIndefiniteLengthMap); + } + + void HandleMapEnd() override { + if (!status_->ok()) + return; + out_->push_back(kStopByte); + assert(!envelopes_.empty()); + if (!envelopes_.back().EncodeStop(out_)) { + HandleError( + Status(Error::CBOR_ENVELOPE_SIZE_LIMIT_EXCEEDED, out_->size())); + return; + } + envelopes_.pop_back(); + } + + void HandleArrayBegin() override { + if (!status_->ok()) + return; + envelopes_.emplace_back(); + envelopes_.back().EncodeStart(out_); + out_->push_back(kInitialByteIndefiniteLengthArray); + } + + void HandleArrayEnd() override { + if (!status_->ok()) + return; + out_->push_back(kStopByte); + assert(!envelopes_.empty()); + if (!envelopes_.back().EncodeStop(out_)) { + HandleError( + Status(Error::CBOR_ENVELOPE_SIZE_LIMIT_EXCEEDED, out_->size())); + return; + } + envelopes_.pop_back(); + } + + void HandleString8(span chars) override { + if (!status_->ok()) + return; + EncodeString8(chars, out_); + } + + void HandleString16(span chars) override { + if (!status_->ok()) + return; + EncodeFromUTF16(chars, out_); + } + + void HandleBinary(span bytes) override { + if (!status_->ok()) + return; + EncodeBinary(bytes, out_); + } + + void HandleDouble(double value) override { + if (!status_->ok()) + return; + EncodeDouble(value, out_); + } + + void HandleInt32(int32_t value) override { + if (!status_->ok()) + return; + EncodeInt32(value, out_); + } + + void HandleBool(bool value) override { + if (!status_->ok()) + return; + // See RFC 7049 Section 2.3, Table 2. + out_->push_back(value ? kEncodedTrue : kEncodedFalse); + } + + void HandleNull() override { + if (!status_->ok()) + return; + // See RFC 7049 Section 2.3, Table 2. + out_->push_back(kEncodedNull); + } + + void HandleError(Status error) override { + if (!status_->ok()) + return; + *status_ = error; + out_->clear(); + } + + private: + C* out_; + std::vector envelopes_; + Status* status_; +}; +} // namespace + +std::unique_ptr NewCBOREncoder( + std::vector* out, + Status* status) { + return std::unique_ptr( + new CBOREncoder>(out, status)); +} +std::unique_ptr NewCBOREncoder(std::string* out, + Status* status) { + return std::unique_ptr( + new CBOREncoder(out, status)); +} + +// ============================================================================= +// cbor::CBORTokenizer - for parsing individual CBOR items +// ============================================================================= + +CBORTokenizer::CBORTokenizer(span bytes) : bytes_(bytes) { + ReadNextToken(/*enter_envelope=*/false); +} +CBORTokenizer::~CBORTokenizer() {} + +CBORTokenTag CBORTokenizer::TokenTag() const { + return token_tag_; +} + +void CBORTokenizer::Next() { + if (token_tag_ == CBORTokenTag::ERROR_VALUE || + token_tag_ == CBORTokenTag::DONE) + return; + ReadNextToken(/*enter_envelope=*/false); +} + +void CBORTokenizer::EnterEnvelope() { + assert(token_tag_ == CBORTokenTag::ENVELOPE); + ReadNextToken(/*enter_envelope=*/true); +} + +Status CBORTokenizer::Status() const { + return status_; +} + +// The following accessor functions ::GetInt32, ::GetDouble, +// ::GetString8, ::GetString16WireRep, ::GetBinary, ::GetEnvelopeContents +// assume that a particular token was recognized in ::ReadNextToken. +// That's where all the error checking is done. By design, +// the accessors (assuming the token was recognized) never produce +// an error. + +int32_t CBORTokenizer::GetInt32() const { + assert(token_tag_ == CBORTokenTag::INT32); + // The range checks happen in ::ReadNextToken(). + return static_cast( + token_start_type_ == MajorType::UNSIGNED + ? token_start_internal_value_ + : -static_cast(token_start_internal_value_) - 1); +} + +double CBORTokenizer::GetDouble() const { + assert(token_tag_ == CBORTokenTag::DOUBLE); + union { + uint64_t from_uint64; + double to_double; + } reinterpret; + reinterpret.from_uint64 = ReadBytesMostSignificantByteFirst( + bytes_.subspan(status_.pos + 1)); + return reinterpret.to_double; +} + +span CBORTokenizer::GetString8() const { + assert(token_tag_ == CBORTokenTag::STRING8); + auto length = static_cast(token_start_internal_value_); + return bytes_.subspan(status_.pos + (token_byte_length_ - length), length); +} + +span CBORTokenizer::GetString16WireRep() const { + assert(token_tag_ == CBORTokenTag::STRING16); + auto length = static_cast(token_start_internal_value_); + return bytes_.subspan(status_.pos + (token_byte_length_ - length), length); +} + +span CBORTokenizer::GetBinary() const { + assert(token_tag_ == CBORTokenTag::BINARY); + auto length = static_cast(token_start_internal_value_); + return bytes_.subspan(status_.pos + (token_byte_length_ - length), length); +} + +span CBORTokenizer::GetEnvelopeContents() const { + assert(token_tag_ == CBORTokenTag::ENVELOPE); + auto length = static_cast(token_start_internal_value_); + return bytes_.subspan(status_.pos + kEncodedEnvelopeHeaderSize, length); +} + +// All error checking happens in ::ReadNextToken, so that the accessors +// can avoid having to carry an error return value. +// +// With respect to checking the encoded lengths of strings, arrays, etc: +// On the wire, CBOR uses 1,2,4, and 8 byte unsigned integers, so +// we initially read them as uint64_t, usually into token_start_internal_value_. +// +// However, since these containers have a representation on the machine, +// we need to do corresponding size computations on the input byte array, +// output span (e.g. the payload for a string), etc., and size_t is +// machine specific (in practice either 32 bit or 64 bit). +// +// Further, we must avoid overflowing size_t. Therefore, we use this +// kMaxValidLength constant to: +// - Reject values that are larger than the architecture specific +// max size_t (differs between 32 bit and 64 bit arch). +// - Reserve at least one bit so that we can check against overflows +// when adding lengths (array / string length / etc.); we do this by +// ensuring that the inputs to an addition are <= kMaxValidLength, +// and then checking whether the sum went past it. +// +// See also +// https://chromium.googlesource.com/chromium/src/+/master/docs/security/integer-semantics.md +static const uint64_t kMaxValidLength = + std::min(std::numeric_limits::max() >> 2, + std::numeric_limits::max()); + +void CBORTokenizer::ReadNextToken(bool enter_envelope) { + if (enter_envelope) { + status_.pos += kEncodedEnvelopeHeaderSize; + } else { + status_.pos = + status_.pos == Status::npos() ? 0 : status_.pos + token_byte_length_; + } + status_.error = Error::OK; + if (status_.pos >= bytes_.size()) { + token_tag_ = CBORTokenTag::DONE; + return; + } + const size_t remaining_bytes = bytes_.size() - status_.pos; + switch (bytes_[status_.pos]) { + case kStopByte: + SetToken(CBORTokenTag::STOP, 1); + return; + case kInitialByteIndefiniteLengthMap: + SetToken(CBORTokenTag::MAP_START, 1); + return; + case kInitialByteIndefiniteLengthArray: + SetToken(CBORTokenTag::ARRAY_START, 1); + return; + case kEncodedTrue: + SetToken(CBORTokenTag::TRUE_VALUE, 1); + return; + case kEncodedFalse: + SetToken(CBORTokenTag::FALSE_VALUE, 1); + return; + case kEncodedNull: + SetToken(CBORTokenTag::NULL_VALUE, 1); + return; + case kExpectedConversionToBase64Tag: { // BINARY + const int8_t bytes_read = internals::ReadTokenStart( + bytes_.subspan(status_.pos + 1), &token_start_type_, + &token_start_internal_value_); + if (bytes_read < 0 || token_start_type_ != MajorType::BYTE_STRING || + token_start_internal_value_ > kMaxValidLength) { + SetError(Error::CBOR_INVALID_BINARY); + return; + } + const uint64_t token_byte_length = token_start_internal_value_ + + /* tag before token start: */ 1 + + /* token start: */ bytes_read; + if (token_byte_length > remaining_bytes) { + SetError(Error::CBOR_INVALID_BINARY); + return; + } + SetToken(CBORTokenTag::BINARY, static_cast(token_byte_length)); + return; + } + case kInitialByteForDouble: { // DOUBLE + if (kEncodedDoubleSize > remaining_bytes) { + SetError(Error::CBOR_INVALID_DOUBLE); + return; + } + SetToken(CBORTokenTag::DOUBLE, kEncodedDoubleSize); + return; + } + case kInitialByteForEnvelope: { // ENVELOPE + if (kEncodedEnvelopeHeaderSize > remaining_bytes) { + SetError(Error::CBOR_INVALID_ENVELOPE); + return; + } + // The envelope must be a byte string with 32 bit length. + if (bytes_[status_.pos + 1] != kInitialByteFor32BitLengthByteString) { + SetError(Error::CBOR_INVALID_ENVELOPE); + return; + } + // Read the length of the byte string. + token_start_internal_value_ = ReadBytesMostSignificantByteFirst( + bytes_.subspan(status_.pos + 2)); + if (token_start_internal_value_ > kMaxValidLength) { + SetError(Error::CBOR_INVALID_ENVELOPE); + return; + } + uint64_t token_byte_length = + token_start_internal_value_ + kEncodedEnvelopeHeaderSize; + if (token_byte_length > remaining_bytes) { + SetError(Error::CBOR_INVALID_ENVELOPE); + return; + } + SetToken(CBORTokenTag::ENVELOPE, static_cast(token_byte_length)); + return; + } + default: { + const int8_t token_start_length = internals::ReadTokenStart( + bytes_.subspan(status_.pos), &token_start_type_, + &token_start_internal_value_); + const bool success = token_start_length >= 0; + switch (token_start_type_) { + case MajorType::UNSIGNED: // INT32. + // INT32 is a signed int32 (int32 makes sense for the + // inspector_protocol, it's not a CBOR limitation), so we check + // against the signed max, so that the allowable values are + // 0, 1, 2, ... 2^31 - 1. + if (!success || std::numeric_limits::max() < + token_start_internal_value_) { + SetError(Error::CBOR_INVALID_INT32); + return; + } + SetToken(CBORTokenTag::INT32, token_start_length); + return; + case MajorType::NEGATIVE: { // INT32. + // INT32 is a signed int32 (int32 makes sense for the + // inspector_protocol, it's not a CBOR limitation); in CBOR, + // the negative values for INT32 are represented as NEGATIVE, + // that is, -1 INT32 is represented as 1 << 5 | 0 (major type 1, + // additional info value 0). So here, we compute the INT32 value + // and then check it against the INT32 min. + int64_t actual_value = + -static_cast(token_start_internal_value_) - 1; + if (!success || actual_value < std::numeric_limits::min()) { + SetError(Error::CBOR_INVALID_INT32); + return; + } + SetToken(CBORTokenTag::INT32, token_start_length); + return; + } + case MajorType::STRING: { // STRING8. + if (!success || token_start_internal_value_ > kMaxValidLength) { + SetError(Error::CBOR_INVALID_STRING8); + return; + } + uint64_t token_byte_length = + token_start_internal_value_ + token_start_length; + if (token_byte_length > remaining_bytes) { + SetError(Error::CBOR_INVALID_STRING8); + return; + } + SetToken(CBORTokenTag::STRING8, + static_cast(token_byte_length)); + return; + } + case MajorType::BYTE_STRING: { // STRING16. + // Length must be divisible by 2 since UTF16 is 2 bytes per + // character, hence the &1 check. + if (!success || token_start_internal_value_ > kMaxValidLength || + token_start_internal_value_ & 1) { + SetError(Error::CBOR_INVALID_STRING16); + return; + } + uint64_t token_byte_length = + token_start_internal_value_ + token_start_length; + if (token_byte_length > remaining_bytes) { + SetError(Error::CBOR_INVALID_STRING16); + return; + } + SetToken(CBORTokenTag::STRING16, + static_cast(token_byte_length)); + return; + } + case MajorType::ARRAY: + case MajorType::MAP: + case MajorType::TAG: + case MajorType::SIMPLE_VALUE: + SetError(Error::CBOR_UNSUPPORTED_VALUE); + return; + } + } + } +} + +void CBORTokenizer::SetToken(CBORTokenTag token_tag, size_t token_byte_length) { + token_tag_ = token_tag; + token_byte_length_ = token_byte_length; +} + +void CBORTokenizer::SetError(Error error) { + token_tag_ = CBORTokenTag::ERROR_VALUE; + status_.error = error; +} + +// ============================================================================= +// cbor::ParseCBOR - for receiving streaming parser events for CBOR messages +// ============================================================================= + +namespace { +// When parsing CBOR, we limit recursion depth for objects and arrays +// to this constant. +static constexpr int kStackLimit = 300; + +// Below are three parsing routines for CBOR, which cover enough +// to roundtrip JSON messages. +bool ParseMap(int32_t stack_depth, + CBORTokenizer* tokenizer, + StreamingParserHandler* out); +bool ParseArray(int32_t stack_depth, + CBORTokenizer* tokenizer, + StreamingParserHandler* out); +bool ParseValue(int32_t stack_depth, + CBORTokenizer* tokenizer, + StreamingParserHandler* out); + +void ParseUTF16String(CBORTokenizer* tokenizer, StreamingParserHandler* out) { + std::vector value; + span rep = tokenizer->GetString16WireRep(); + for (size_t ii = 0; ii < rep.size(); ii += 2) + value.push_back((rep[ii + 1] << 8) | rep[ii]); + out->HandleString16(span(value.data(), value.size())); + tokenizer->Next(); +} + +bool ParseUTF8String(CBORTokenizer* tokenizer, StreamingParserHandler* out) { + assert(tokenizer->TokenTag() == CBORTokenTag::STRING8); + out->HandleString8(tokenizer->GetString8()); + tokenizer->Next(); + return true; +} + +bool ParseValue(int32_t stack_depth, + CBORTokenizer* tokenizer, + StreamingParserHandler* out) { + if (stack_depth > kStackLimit) { + out->HandleError( + Status{Error::CBOR_STACK_LIMIT_EXCEEDED, tokenizer->Status().pos}); + return false; + } + // Skip past the envelope to get to what's inside. + if (tokenizer->TokenTag() == CBORTokenTag::ENVELOPE) + tokenizer->EnterEnvelope(); + switch (tokenizer->TokenTag()) { + case CBORTokenTag::ERROR_VALUE: + out->HandleError(tokenizer->Status()); + return false; + case CBORTokenTag::DONE: + out->HandleError(Status{Error::CBOR_UNEXPECTED_EOF_EXPECTED_VALUE, + tokenizer->Status().pos}); + return false; + case CBORTokenTag::TRUE_VALUE: + out->HandleBool(true); + tokenizer->Next(); + return true; + case CBORTokenTag::FALSE_VALUE: + out->HandleBool(false); + tokenizer->Next(); + return true; + case CBORTokenTag::NULL_VALUE: + out->HandleNull(); + tokenizer->Next(); + return true; + case CBORTokenTag::INT32: + out->HandleInt32(tokenizer->GetInt32()); + tokenizer->Next(); + return true; + case CBORTokenTag::DOUBLE: + out->HandleDouble(tokenizer->GetDouble()); + tokenizer->Next(); + return true; + case CBORTokenTag::STRING8: + return ParseUTF8String(tokenizer, out); + case CBORTokenTag::STRING16: + ParseUTF16String(tokenizer, out); + return true; + case CBORTokenTag::BINARY: { + out->HandleBinary(tokenizer->GetBinary()); + tokenizer->Next(); + return true; + } + case CBORTokenTag::MAP_START: + return ParseMap(stack_depth + 1, tokenizer, out); + case CBORTokenTag::ARRAY_START: + return ParseArray(stack_depth + 1, tokenizer, out); + default: + out->HandleError( + Status{Error::CBOR_UNSUPPORTED_VALUE, tokenizer->Status().pos}); + return false; + } +} + +// |bytes| must start with the indefinite length array byte, so basically, +// ParseArray may only be called after an indefinite length array has been +// detected. +bool ParseArray(int32_t stack_depth, + CBORTokenizer* tokenizer, + StreamingParserHandler* out) { + assert(tokenizer->TokenTag() == CBORTokenTag::ARRAY_START); + tokenizer->Next(); + out->HandleArrayBegin(); + while (tokenizer->TokenTag() != CBORTokenTag::STOP) { + if (tokenizer->TokenTag() == CBORTokenTag::DONE) { + out->HandleError( + Status{Error::CBOR_UNEXPECTED_EOF_IN_ARRAY, tokenizer->Status().pos}); + return false; + } + if (tokenizer->TokenTag() == CBORTokenTag::ERROR_VALUE) { + out->HandleError(tokenizer->Status()); + return false; + } + // Parse value. + if (!ParseValue(stack_depth, tokenizer, out)) + return false; + } + out->HandleArrayEnd(); + tokenizer->Next(); + return true; +} + +// |bytes| must start with the indefinite length array byte, so basically, +// ParseArray may only be called after an indefinite length array has been +// detected. +bool ParseMap(int32_t stack_depth, + CBORTokenizer* tokenizer, + StreamingParserHandler* out) { + assert(tokenizer->TokenTag() == CBORTokenTag::MAP_START); + out->HandleMapBegin(); + tokenizer->Next(); + while (tokenizer->TokenTag() != CBORTokenTag::STOP) { + if (tokenizer->TokenTag() == CBORTokenTag::DONE) { + out->HandleError( + Status{Error::CBOR_UNEXPECTED_EOF_IN_MAP, tokenizer->Status().pos}); + return false; + } + if (tokenizer->TokenTag() == CBORTokenTag::ERROR_VALUE) { + out->HandleError(tokenizer->Status()); + return false; + } + // Parse key. + if (tokenizer->TokenTag() == CBORTokenTag::STRING8) { + if (!ParseUTF8String(tokenizer, out)) + return false; + } else if (tokenizer->TokenTag() == CBORTokenTag::STRING16) { + ParseUTF16String(tokenizer, out); + } else { + out->HandleError( + Status{Error::CBOR_INVALID_MAP_KEY, tokenizer->Status().pos}); + return false; + } + // Parse value. + if (!ParseValue(stack_depth, tokenizer, out)) + return false; + } + out->HandleMapEnd(); + tokenizer->Next(); + return true; +} +} // namespace + +void ParseCBOR(span bytes, StreamingParserHandler* out) { + if (bytes.empty()) { + out->HandleError(Status{Error::CBOR_NO_INPUT, 0}); + return; + } + if (bytes[0] != kInitialByteForEnvelope) { + out->HandleError(Status{Error::CBOR_INVALID_START_BYTE, 0}); + return; + } + CBORTokenizer tokenizer(bytes); + if (tokenizer.TokenTag() == CBORTokenTag::ERROR_VALUE) { + out->HandleError(tokenizer.Status()); + return; + } + // We checked for the envelope start byte above, so the tokenizer + // must agree here, since it's not an error. + assert(tokenizer.TokenTag() == CBORTokenTag::ENVELOPE); + tokenizer.EnterEnvelope(); + if (tokenizer.TokenTag() != CBORTokenTag::MAP_START) { + out->HandleError( + Status{Error::CBOR_MAP_START_EXPECTED, tokenizer.Status().pos}); + return; + } + if (!ParseMap(/*stack_depth=*/1, &tokenizer, out)) + return; + if (tokenizer.TokenTag() == CBORTokenTag::DONE) + return; + if (tokenizer.TokenTag() == CBORTokenTag::ERROR_VALUE) { + out->HandleError(tokenizer.Status()); + return; + } + out->HandleError(Status{Error::CBOR_TRAILING_JUNK, tokenizer.Status().pos}); +} + +// ============================================================================= +// cbor::AppendString8EntryToMap - for limited in-place editing of messages +// ============================================================================= + +template +Status AppendString8EntryToCBORMapTmpl(span string8_key, + span string8_value, + C* cbor) { + // Careful below: Don't compare (*cbor)[idx] with a uint8_t, since + // it could be a char (signed!). Instead, use bytes. + span bytes(reinterpret_cast(cbor->data()), + cbor->size()); + CBORTokenizer tokenizer(bytes); + if (tokenizer.TokenTag() == CBORTokenTag::ERROR_VALUE) + return tokenizer.Status(); + if (tokenizer.TokenTag() != CBORTokenTag::ENVELOPE) + return Status(Error::CBOR_INVALID_ENVELOPE, 0); + size_t envelope_size = tokenizer.GetEnvelopeContents().size(); + size_t old_size = cbor->size(); + if (old_size != envelope_size + kEncodedEnvelopeHeaderSize) + return Status(Error::CBOR_INVALID_ENVELOPE, 0); + if (envelope_size == 0 || + (tokenizer.GetEnvelopeContents()[0] != EncodeIndefiniteLengthMapStart())) + return Status(Error::CBOR_MAP_START_EXPECTED, kEncodedEnvelopeHeaderSize); + if (bytes[bytes.size() - 1] != EncodeStop()) + return Status(Error::CBOR_MAP_STOP_EXPECTED, cbor->size() - 1); + cbor->pop_back(); + EncodeString8(string8_key, cbor); + EncodeString8(string8_value, cbor); + cbor->push_back(EncodeStop()); + size_t new_envelope_size = envelope_size + (cbor->size() - old_size); + if (new_envelope_size > std::numeric_limits::max()) + return Status(Error::CBOR_ENVELOPE_SIZE_LIMIT_EXCEEDED, 0); + size_t size_pos = cbor->size() - new_envelope_size - sizeof(uint32_t); + uint8_t* out = reinterpret_cast(&cbor->at(size_pos)); + *(out++) = (new_envelope_size >> 24) & 0xff; + *(out++) = (new_envelope_size >> 16) & 0xff; + *(out++) = (new_envelope_size >> 8) & 0xff; + *(out) = new_envelope_size & 0xff; + return Status(); +} +Status AppendString8EntryToCBORMap(span string8_key, + span string8_value, + std::vector* cbor) { + return AppendString8EntryToCBORMapTmpl(string8_key, string8_value, cbor); +} +Status AppendString8EntryToCBORMap(span string8_key, + span string8_value, + std::string* cbor) { + return AppendString8EntryToCBORMapTmpl(string8_key, string8_value, cbor); +} +} // namespace cbor + +namespace json { + +// ============================================================================= +// json::NewJSONEncoder - for encoding streaming parser events as JSON +// ============================================================================= + +namespace { +// Prints |value| to |out| with 4 hex digits, most significant chunk first. +template +void PrintHex(uint16_t value, C* out) { + for (int ii = 3; ii >= 0; --ii) { + int four_bits = 0xf & (value >> (4 * ii)); + out->push_back(four_bits + ((four_bits <= 9) ? '0' : ('a' - 10))); + } +} + +// In the writer below, we maintain a stack of State instances. +// It is just enough to emit the appropriate delimiters and brackets +// in JSON. +enum class Container { + // Used for the top-level, initial state. + NONE, + // Inside a JSON object. + MAP, + // Inside a JSON array. + ARRAY +}; +class State { + public: + explicit State(Container container) : container_(container) {} + void StartElement(std::vector* out) { StartElementTmpl(out); } + void StartElement(std::string* out) { StartElementTmpl(out); } + Container container() const { return container_; } + + private: + template + void StartElementTmpl(C* out) { + assert(container_ != Container::NONE || size_ == 0); + if (size_ != 0) { + char delim = (!(size_ & 1) || container_ == Container::ARRAY) ? ',' : ':'; + out->push_back(delim); + } + ++size_; + } + + Container container_ = Container::NONE; + int size_ = 0; +}; + +constexpr char kBase64Table[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz0123456789+/"; + +template +void Base64Encode(const span& in, C* out) { + // The following three cases are based on the tables in the example + // section in https://en.wikipedia.org/wiki/Base64. We process three + // input bytes at a time, emitting 4 output bytes at a time. + size_t ii = 0; + + // While possible, process three input bytes. + for (; ii + 3 <= in.size(); ii += 3) { + uint32_t twentyfour_bits = (in[ii] << 16) | (in[ii + 1] << 8) | in[ii + 2]; + out->push_back(kBase64Table[(twentyfour_bits >> 18)]); + out->push_back(kBase64Table[(twentyfour_bits >> 12) & 0x3f]); + out->push_back(kBase64Table[(twentyfour_bits >> 6) & 0x3f]); + out->push_back(kBase64Table[twentyfour_bits & 0x3f]); + } + if (ii + 2 <= in.size()) { // Process two input bytes. + uint32_t twentyfour_bits = (in[ii] << 16) | (in[ii + 1] << 8); + out->push_back(kBase64Table[(twentyfour_bits >> 18)]); + out->push_back(kBase64Table[(twentyfour_bits >> 12) & 0x3f]); + out->push_back(kBase64Table[(twentyfour_bits >> 6) & 0x3f]); + out->push_back('='); // Emit padding. + return; + } + if (ii + 1 <= in.size()) { // Process a single input byte. + uint32_t twentyfour_bits = (in[ii] << 16); + out->push_back(kBase64Table[(twentyfour_bits >> 18)]); + out->push_back(kBase64Table[(twentyfour_bits >> 12) & 0x3f]); + out->push_back('='); // Emit padding. + out->push_back('='); // Emit padding. + } +} + +// Implements a handler for JSON parser events to emit a JSON string. +template +class JSONEncoder : public StreamingParserHandler { + public: + JSONEncoder(const Platform* platform, C* out, Status* status) + : platform_(platform), out_(out), status_(status) { + *status_ = Status(); + state_.emplace(Container::NONE); + } + + void HandleMapBegin() override { + if (!status_->ok()) + return; + assert(!state_.empty()); + state_.top().StartElement(out_); + state_.emplace(Container::MAP); + Emit('{'); + } + + void HandleMapEnd() override { + if (!status_->ok()) + return; + assert(state_.size() >= 2 && state_.top().container() == Container::MAP); + state_.pop(); + Emit('}'); + } + + void HandleArrayBegin() override { + if (!status_->ok()) + return; + state_.top().StartElement(out_); + state_.emplace(Container::ARRAY); + Emit('['); + } + + void HandleArrayEnd() override { + if (!status_->ok()) + return; + assert(state_.size() >= 2 && state_.top().container() == Container::ARRAY); + state_.pop(); + Emit(']'); + } + + void HandleString16(span chars) override { + if (!status_->ok()) + return; + state_.top().StartElement(out_); + Emit('"'); + for (const uint16_t ch : chars) { + if (ch == '"') { + Emit("\\\""); + } else if (ch == '\\') { + Emit("\\\\"); + } else if (ch == '\b') { + Emit("\\b"); + } else if (ch == '\f') { + Emit("\\f"); + } else if (ch == '\n') { + Emit("\\n"); + } else if (ch == '\r') { + Emit("\\r"); + } else if (ch == '\t') { + Emit("\\t"); + } else if (ch >= 32 && ch <= 126) { + Emit(ch); + } else { + Emit("\\u"); + PrintHex(ch, out_); + } + } + Emit('"'); + } + + void HandleString8(span chars) override { + if (!status_->ok()) + return; + state_.top().StartElement(out_); + Emit('"'); + for (size_t ii = 0; ii < chars.size(); ++ii) { + uint8_t c = chars[ii]; + if (c == '"') { + Emit("\\\""); + } else if (c == '\\') { + Emit("\\\\"); + } else if (c == '\b') { + Emit("\\b"); + } else if (c == '\f') { + Emit("\\f"); + } else if (c == '\n') { + Emit("\\n"); + } else if (c == '\r') { + Emit("\\r"); + } else if (c == '\t') { + Emit("\\t"); + } else if (c >= 32 && c <= 126) { + Emit(c); + } else if (c < 32) { + Emit("\\u"); + PrintHex(static_cast(c), out_); + } else { + // Inspect the leading byte to figure out how long the utf8 + // byte sequence is; while doing this initialize |codepoint| + // with the first few bits. + // See table in: https://en.wikipedia.org/wiki/UTF-8 + // byte one is 110x xxxx -> 2 byte utf8 sequence + // byte one is 1110 xxxx -> 3 byte utf8 sequence + // byte one is 1111 0xxx -> 4 byte utf8 sequence + uint32_t codepoint; + int num_bytes_left; + if ((c & 0xe0) == 0xc0) { // 2 byte utf8 sequence + num_bytes_left = 1; + codepoint = c & 0x1f; + } else if ((c & 0xf0) == 0xe0) { // 3 byte utf8 sequence + num_bytes_left = 2; + codepoint = c & 0x0f; + } else if ((c & 0xf8) == 0xf0) { // 4 byte utf8 sequence + codepoint = c & 0x07; + num_bytes_left = 3; + } else { + continue; // invalid leading byte + } + + // If we have enough bytes in our input, decode the remaining ones + // belonging to this Unicode character into |codepoint|. + if (ii + num_bytes_left > chars.size()) + continue; + while (num_bytes_left > 0) { + c = chars[++ii]; + --num_bytes_left; + // Check the next byte is a continuation byte, that is 10xx xxxx. + if ((c & 0xc0) != 0x80) + continue; + codepoint = (codepoint << 6) | (c & 0x3f); + } + + // Disallow overlong encodings for ascii characters, as these + // would include " and other characters significant to JSON + // string termination / control. + if (codepoint < 0x7f) + continue; + // Invalid in UTF8, and can't be represented in UTF16 anyway. + if (codepoint > 0x10ffff) + continue; + + // So, now we transcode to UTF16, + // using the math described at https://en.wikipedia.org/wiki/UTF-16, + // for either one or two 16 bit characters. + if (codepoint < 0xffff) { + Emit("\\u"); + PrintHex(static_cast(codepoint), out_); + continue; + } + codepoint -= 0x10000; + // high surrogate + Emit("\\u"); + PrintHex(static_cast((codepoint >> 10) + 0xd800), out_); + // low surrogate + Emit("\\u"); + PrintHex(static_cast((codepoint & 0x3ff) + 0xdc00), out_); + } + } + Emit('"'); + } + + void HandleBinary(span bytes) override { + if (!status_->ok()) + return; + state_.top().StartElement(out_); + Emit('"'); + Base64Encode(bytes, out_); + Emit('"'); + } + + void HandleDouble(double value) override { + if (!status_->ok()) + return; + state_.top().StartElement(out_); + // JSON cannot represent NaN or Infinity. So, for compatibility, + // we behave like the JSON object in web browsers: emit 'null'. + if (!std::isfinite(value)) { + Emit("null"); + return; + } + std::unique_ptr str_value = platform_->DToStr(value); + + // DToStr may fail to emit a 0 before the decimal dot. E.g. this is + // the case in base::NumberToString in Chromium (which is based on + // dmg_fp). So, much like + // https://cs.chromium.org/chromium/src/base/json/json_writer.cc + // we probe for this and emit the leading 0 anyway if necessary. + const char* chars = str_value.get(); + if (chars[0] == '.') { + Emit('0'); + } else if (chars[0] == '-' && chars[1] == '.') { + Emit("-0"); + ++chars; + } + Emit(chars); + } + + void HandleInt32(int32_t value) override { + if (!status_->ok()) + return; + state_.top().StartElement(out_); + Emit(std::to_string(value)); + } + + void HandleBool(bool value) override { + if (!status_->ok()) + return; + state_.top().StartElement(out_); + Emit(value ? "true" : "false"); + } + + void HandleNull() override { + if (!status_->ok()) + return; + state_.top().StartElement(out_); + Emit("null"); + } + + void HandleError(Status error) override { + assert(!error.ok()); + *status_ = error; + out_->clear(); + } + + private: + void Emit(char c) { out_->push_back(c); } + void Emit(const char* str) { + out_->insert(out_->end(), str, str + strlen(str)); + } + void Emit(const std::string& str) { + out_->insert(out_->end(), str.begin(), str.end()); + } + + const Platform* platform_; + C* out_; + Status* status_; + std::stack state_; +}; +} // namespace + +std::unique_ptr NewJSONEncoder( + const Platform* platform, + std::vector* out, + Status* status) { + return std::unique_ptr( + new JSONEncoder>(platform, out, status)); +} +std::unique_ptr NewJSONEncoder(const Platform* platform, + std::string* out, + Status* status) { + return std::unique_ptr( + new JSONEncoder(platform, out, status)); +} + +// ============================================================================= +// json::ParseJSON - for receiving streaming parser events for JSON. +// ============================================================================= + +namespace { +const int kStackLimit = 300; + +enum Token { + ObjectBegin, + ObjectEnd, + ArrayBegin, + ArrayEnd, + StringLiteral, + Number, + BoolTrue, + BoolFalse, + NullToken, + ListSeparator, + ObjectPairSeparator, + InvalidToken, + NoInput +}; + +const char* const kNullString = "null"; +const char* const kTrueString = "true"; +const char* const kFalseString = "false"; + +template +class JsonParser { + public: + JsonParser(const Platform* platform, StreamingParserHandler* handler) + : platform_(platform), handler_(handler) {} + + void Parse(const Char* start, size_t length) { + start_pos_ = start; + const Char* end = start + length; + const Char* tokenEnd = nullptr; + ParseValue(start, end, &tokenEnd, 0); + if (error_) + return; + if (tokenEnd != end) { + HandleError(Error::JSON_PARSER_UNPROCESSED_INPUT_REMAINS, tokenEnd); + } + } + + private: + bool CharsToDouble(const uint16_t* chars, size_t length, double* result) { + std::string buffer; + buffer.reserve(length + 1); + for (size_t ii = 0; ii < length; ++ii) { + bool is_ascii = !(chars[ii] & ~0x7F); + if (!is_ascii) + return false; + buffer.push_back(static_cast(chars[ii])); + } + return platform_->StrToD(buffer.c_str(), result); + } + + bool CharsToDouble(const uint8_t* chars, size_t length, double* result) { + std::string buffer(reinterpret_cast(chars), length); + return platform_->StrToD(buffer.c_str(), result); + } + + static bool ParseConstToken(const Char* start, + const Char* end, + const Char** token_end, + const char* token) { + // |token| is \0 terminated, it's one of the constants at top of the file. + while (start < end && *token != '\0' && *start++ == *token++) { + } + if (*token != '\0') + return false; + *token_end = start; + return true; + } + + static bool ReadInt(const Char* start, + const Char* end, + const Char** token_end, + bool allow_leading_zeros) { + if (start == end) + return false; + bool has_leading_zero = '0' == *start; + int length = 0; + while (start < end && '0' <= *start && *start <= '9') { + ++start; + ++length; + } + if (!length) + return false; + if (!allow_leading_zeros && length > 1 && has_leading_zero) + return false; + *token_end = start; + return true; + } + + static bool ParseNumberToken(const Char* start, + const Char* end, + const Char** token_end) { + // We just grab the number here. We validate the size in DecodeNumber. + // According to RFC4627, a valid number is: [minus] int [frac] [exp] + if (start == end) + return false; + Char c = *start; + if ('-' == c) + ++start; + + if (!ReadInt(start, end, &start, /*allow_leading_zeros=*/false)) + return false; + if (start == end) { + *token_end = start; + return true; + } + + // Optional fraction part + c = *start; + if ('.' == c) { + ++start; + if (!ReadInt(start, end, &start, /*allow_leading_zeros=*/true)) + return false; + if (start == end) { + *token_end = start; + return true; + } + c = *start; + } + + // Optional exponent part + if ('e' == c || 'E' == c) { + ++start; + if (start == end) + return false; + c = *start; + if ('-' == c || '+' == c) { + ++start; + if (start == end) + return false; + } + if (!ReadInt(start, end, &start, /*allow_leading_zeros=*/true)) + return false; + } + + *token_end = start; + return true; + } + + static bool ReadHexDigits(const Char* start, + const Char* end, + const Char** token_end, + int digits) { + if (end - start < digits) + return false; + for (int i = 0; i < digits; ++i) { + Char c = *start++; + if (!(('0' <= c && c <= '9') || ('a' <= c && c <= 'f') || + ('A' <= c && c <= 'F'))) + return false; + } + *token_end = start; + return true; + } + + static bool ParseStringToken(const Char* start, + const Char* end, + const Char** token_end) { + while (start < end) { + Char c = *start++; + if ('\\' == c) { + if (start == end) + return false; + c = *start++; + // Make sure the escaped char is valid. + switch (c) { + case 'x': + if (!ReadHexDigits(start, end, &start, 2)) + return false; + break; + case 'u': + if (!ReadHexDigits(start, end, &start, 4)) + return false; + break; + case '\\': + case '/': + case 'b': + case 'f': + case 'n': + case 'r': + case 't': + case 'v': + case '"': + break; + default: + return false; + } + } else if ('"' == c) { + *token_end = start; + return true; + } + } + return false; + } + + static bool SkipComment(const Char* start, + const Char* end, + const Char** comment_end) { + if (start == end) + return false; + + if (*start != '/' || start + 1 >= end) + return false; + ++start; + + if (*start == '/') { + // Single line comment, read to newline. + for (++start; start < end; ++start) { + if (*start == '\n' || *start == '\r') { + *comment_end = start + 1; + return true; + } + } + *comment_end = end; + // Comment reaches end-of-input, which is fine. + return true; + } + + if (*start == '*') { + Char previous = '\0'; + // Block comment, read until end marker. + for (++start; start < end; previous = *start++) { + if (previous == '*' && *start == '/') { + *comment_end = start + 1; + return true; + } + } + // Block comment must close before end-of-input. + return false; + } + + return false; + } + + static bool IsSpaceOrNewLine(Char c) { + // \v = vertial tab; \f = form feed page break. + return c == ' ' || c == '\n' || c == '\v' || c == '\f' || c == '\r' || + c == '\t'; + } + + static void SkipWhitespaceAndComments(const Char* start, + const Char* end, + const Char** whitespace_end) { + while (start < end) { + if (IsSpaceOrNewLine(*start)) { + ++start; + } else if (*start == '/') { + const Char* comment_end = nullptr; + if (!SkipComment(start, end, &comment_end)) + break; + start = comment_end; + } else { + break; + } + } + *whitespace_end = start; + } + + static Token ParseToken(const Char* start, + const Char* end, + const Char** tokenStart, + const Char** token_end) { + SkipWhitespaceAndComments(start, end, tokenStart); + start = *tokenStart; + + if (start == end) + return NoInput; + + switch (*start) { + case 'n': + if (ParseConstToken(start, end, token_end, kNullString)) + return NullToken; + break; + case 't': + if (ParseConstToken(start, end, token_end, kTrueString)) + return BoolTrue; + break; + case 'f': + if (ParseConstToken(start, end, token_end, kFalseString)) + return BoolFalse; + break; + case '[': + *token_end = start + 1; + return ArrayBegin; + case ']': + *token_end = start + 1; + return ArrayEnd; + case ',': + *token_end = start + 1; + return ListSeparator; + case '{': + *token_end = start + 1; + return ObjectBegin; + case '}': + *token_end = start + 1; + return ObjectEnd; + case ':': + *token_end = start + 1; + return ObjectPairSeparator; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case '-': + if (ParseNumberToken(start, end, token_end)) + return Number; + break; + case '"': + if (ParseStringToken(start + 1, end, token_end)) + return StringLiteral; + break; + } + return InvalidToken; + } + + static int HexToInt(Char c) { + if ('0' <= c && c <= '9') + return c - '0'; + if ('A' <= c && c <= 'F') + return c - 'A' + 10; + if ('a' <= c && c <= 'f') + return c - 'a' + 10; + assert(false); // Unreachable. + return 0; + } + + static bool DecodeString(const Char* start, + const Char* end, + std::vector* output) { + if (start == end) + return true; + if (start > end) + return false; + output->reserve(end - start); + while (start < end) { + uint16_t c = *start++; + // If the |Char| we're dealing with is really a byte, then + // we have utf8 here, and we need to check for multibyte characters + // and transcode them to utf16 (either one or two utf16 chars). + if (sizeof(Char) == sizeof(uint8_t) && c >= 0x7f) { + // Inspect the leading byte to figure out how long the utf8 + // byte sequence is; while doing this initialize |codepoint| + // with the first few bits. + // See table in: https://en.wikipedia.org/wiki/UTF-8 + // byte one is 110x xxxx -> 2 byte utf8 sequence + // byte one is 1110 xxxx -> 3 byte utf8 sequence + // byte one is 1111 0xxx -> 4 byte utf8 sequence + uint32_t codepoint; + int num_bytes_left; + if ((c & 0xe0) == 0xc0) { // 2 byte utf8 sequence + num_bytes_left = 1; + codepoint = c & 0x1f; + } else if ((c & 0xf0) == 0xe0) { // 3 byte utf8 sequence + num_bytes_left = 2; + codepoint = c & 0x0f; + } else if ((c & 0xf8) == 0xf0) { // 4 byte utf8 sequence + codepoint = c & 0x07; + num_bytes_left = 3; + } else { + return false; // invalid leading byte + } + + // If we have enough bytes in our inpput, decode the remaining ones + // belonging to this Unicode character into |codepoint|. + if (start + num_bytes_left > end) + return false; + while (num_bytes_left > 0) { + c = *start++; + --num_bytes_left; + // Check the next byte is a continuation byte, that is 10xx xxxx. + if ((c & 0xc0) != 0x80) + return false; + codepoint = (codepoint << 6) | (c & 0x3f); + } + + // Disallow overlong encodings for ascii characters, as these + // would include " and other characters significant to JSON + // string termination / control. + if (codepoint < 0x7f) + return false; + // Invalid in UTF8, and can't be represented in UTF16 anyway. + if (codepoint > 0x10ffff) + return false; + + // So, now we transcode to UTF16, + // using the math described at https://en.wikipedia.org/wiki/UTF-16, + // for either one or two 16 bit characters. + if (codepoint < 0xffff) { + output->push_back(codepoint); + continue; + } + codepoint -= 0x10000; + output->push_back((codepoint >> 10) + 0xd800); // high surrogate + output->push_back((codepoint & 0x3ff) + 0xdc00); // low surrogate + continue; + } + if ('\\' != c) { + output->push_back(c); + continue; + } + if (start == end) + return false; + c = *start++; + + if (c == 'x') { + // \x is not supported. + return false; + } + + switch (c) { + case '"': + case '/': + case '\\': + break; + case 'b': + c = '\b'; + break; + case 'f': + c = '\f'; + break; + case 'n': + c = '\n'; + break; + case 'r': + c = '\r'; + break; + case 't': + c = '\t'; + break; + case 'v': + c = '\v'; + break; + case 'u': + c = (HexToInt(*start) << 12) + (HexToInt(*(start + 1)) << 8) + + (HexToInt(*(start + 2)) << 4) + HexToInt(*(start + 3)); + start += 4; + break; + default: + return false; + } + output->push_back(c); + } + return true; + } + + void ParseValue(const Char* start, + const Char* end, + const Char** value_token_end, + int depth) { + if (depth > kStackLimit) { + HandleError(Error::JSON_PARSER_STACK_LIMIT_EXCEEDED, start); + return; + } + const Char* token_start = nullptr; + const Char* token_end = nullptr; + Token token = ParseToken(start, end, &token_start, &token_end); + switch (token) { + case NoInput: + HandleError(Error::JSON_PARSER_NO_INPUT, token_start); + return; + case InvalidToken: + HandleError(Error::JSON_PARSER_INVALID_TOKEN, token_start); + return; + case NullToken: + handler_->HandleNull(); + break; + case BoolTrue: + handler_->HandleBool(true); + break; + case BoolFalse: + handler_->HandleBool(false); + break; + case Number: { + double value; + if (!CharsToDouble(token_start, token_end - token_start, &value)) { + HandleError(Error::JSON_PARSER_INVALID_NUMBER, token_start); + return; + } + if (value >= std::numeric_limits::min() && + value <= std::numeric_limits::max() && + static_cast(value) == value) + handler_->HandleInt32(static_cast(value)); + else + handler_->HandleDouble(value); + break; + } + case StringLiteral: { + std::vector value; + bool ok = DecodeString(token_start + 1, token_end - 1, &value); + if (!ok) { + HandleError(Error::JSON_PARSER_INVALID_STRING, token_start); + return; + } + handler_->HandleString16(span(value.data(), value.size())); + break; + } + case ArrayBegin: { + handler_->HandleArrayBegin(); + start = token_end; + token = ParseToken(start, end, &token_start, &token_end); + while (token != ArrayEnd) { + ParseValue(start, end, &token_end, depth + 1); + if (error_) + return; + + // After a list value, we expect a comma or the end of the list. + start = token_end; + token = ParseToken(start, end, &token_start, &token_end); + if (token == ListSeparator) { + start = token_end; + token = ParseToken(start, end, &token_start, &token_end); + if (token == ArrayEnd) { + HandleError(Error::JSON_PARSER_UNEXPECTED_ARRAY_END, token_start); + return; + } + } else if (token != ArrayEnd) { + // Unexpected value after list value. Bail out. + HandleError(Error::JSON_PARSER_COMMA_OR_ARRAY_END_EXPECTED, + token_start); + return; + } + } + handler_->HandleArrayEnd(); + break; + } + case ObjectBegin: { + handler_->HandleMapBegin(); + start = token_end; + token = ParseToken(start, end, &token_start, &token_end); + while (token != ObjectEnd) { + if (token != StringLiteral) { + HandleError(Error::JSON_PARSER_STRING_LITERAL_EXPECTED, + token_start); + return; + } + std::vector key; + if (!DecodeString(token_start + 1, token_end - 1, &key)) { + HandleError(Error::JSON_PARSER_INVALID_STRING, token_start); + return; + } + handler_->HandleString16(span(key.data(), key.size())); + start = token_end; + + token = ParseToken(start, end, &token_start, &token_end); + if (token != ObjectPairSeparator) { + HandleError(Error::JSON_PARSER_COLON_EXPECTED, token_start); + return; + } + start = token_end; + + ParseValue(start, end, &token_end, depth + 1); + if (error_) + return; + start = token_end; + + // After a key/value pair, we expect a comma or the end of the + // object. + token = ParseToken(start, end, &token_start, &token_end); + if (token == ListSeparator) { + start = token_end; + token = ParseToken(start, end, &token_start, &token_end); + if (token == ObjectEnd) { + HandleError(Error::JSON_PARSER_UNEXPECTED_MAP_END, token_start); + return; + } + } else if (token != ObjectEnd) { + // Unexpected value after last object value. Bail out. + HandleError(Error::JSON_PARSER_COMMA_OR_MAP_END_EXPECTED, + token_start); + return; + } + } + handler_->HandleMapEnd(); + break; + } + + default: + // We got a token that's not a value. + HandleError(Error::JSON_PARSER_VALUE_EXPECTED, token_start); + return; + } + + SkipWhitespaceAndComments(token_end, end, value_token_end); + } + + void HandleError(Error error, const Char* pos) { + assert(error != Error::OK); + if (!error_) { + handler_->HandleError( + Status{error, static_cast(pos - start_pos_)}); + error_ = true; + } + } + + const Char* start_pos_ = nullptr; + bool error_ = false; + const Platform* platform_; + StreamingParserHandler* handler_; +}; +} // namespace + +void ParseJSON(const Platform& platform, + span chars, + StreamingParserHandler* handler) { + JsonParser parser(&platform, handler); + parser.Parse(chars.data(), chars.size()); +} + +void ParseJSON(const Platform& platform, + span chars, + StreamingParserHandler* handler) { + JsonParser parser(&platform, handler); + parser.Parse(chars.data(), chars.size()); +} + +// ============================================================================= +// json::ConvertCBORToJSON, json::ConvertJSONToCBOR - for transcoding +// ============================================================================= +template +Status ConvertCBORToJSONTmpl(const Platform& platform, + span cbor, + C* json) { + Status status; + std::unique_ptr json_writer = + NewJSONEncoder(&platform, json, &status); + cbor::ParseCBOR(cbor, json_writer.get()); + return status; +} + +Status ConvertCBORToJSON(const Platform& platform, + span cbor, + std::vector* json) { + return ConvertCBORToJSONTmpl(platform, cbor, json); +} +Status ConvertCBORToJSON(const Platform& platform, + span cbor, + std::string* json) { + return ConvertCBORToJSONTmpl(platform, cbor, json); +} + +template +Status ConvertJSONToCBORTmpl(const Platform& platform, span json, C* cbor) { + Status status; + std::unique_ptr encoder = + cbor::NewCBOREncoder(cbor, &status); + ParseJSON(platform, json, encoder.get()); + return status; +} +Status ConvertJSONToCBOR(const Platform& platform, + span json, + std::string* cbor) { + return ConvertJSONToCBORTmpl(platform, json, cbor); +} +Status ConvertJSONToCBOR(const Platform& platform, + span json, + std::string* cbor) { + return ConvertJSONToCBORTmpl(platform, json, cbor); +} +Status ConvertJSONToCBOR(const Platform& platform, + span json, + std::vector* cbor) { + return ConvertJSONToCBORTmpl(platform, json, cbor); +} +Status ConvertJSONToCBOR(const Platform& platform, + span json, + std::vector* cbor) { + return ConvertJSONToCBORTmpl(platform, json, cbor); +} +} // namespace json +} // namespace v8_inspector_protocol_encoding diff --git a/tools/inspector_protocol/encoding/encoding.h b/tools/inspector_protocol/encoding/encoding.h new file mode 100644 index 00000000000000..90916d42b36dae --- /dev/null +++ b/tools/inspector_protocol/encoding/encoding.h @@ -0,0 +1,510 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef V8_INSPECTOR_PROTOCOL_ENCODING_ENCODING_H_ +#define V8_INSPECTOR_PROTOCOL_ENCODING_ENCODING_H_ + +#include +#include +#include +#include +#include +#include +#include + +namespace v8_inspector_protocol_encoding { + +// ============================================================================= +// span - sequence of bytes +// ============================================================================= + +// This template is similar to std::span, which will be included in C++20. +template +class span { + public: + using index_type = size_t; + + span() : data_(nullptr), size_(0) {} + span(const T* data, index_type size) : data_(data), size_(size) {} + + const T* data() const { return data_; } + + const T* begin() const { return data_; } + const T* end() const { return data_ + size_; } + + const T& operator[](index_type idx) const { return data_[idx]; } + + span subspan(index_type offset, index_type count) const { + return span(data_ + offset, count); + } + + span subspan(index_type offset) const { + return span(data_ + offset, size_ - offset); + } + + bool empty() const { return size_ == 0; } + + index_type size() const { return size_; } + index_type size_bytes() const { return size_ * sizeof(T); } + + private: + const T* data_; + index_type size_; +}; + +template +span SpanFrom(const std::vector& v) { + return span(v.data(), v.size()); +} + +template +span SpanFrom(const char (&str)[N]) { + return span(reinterpret_cast(str), N - 1); +} + +inline span SpanFrom(const char* str) { + return str ? span(reinterpret_cast(str), strlen(str)) + : span(); +} + +inline span SpanFrom(const std::string& v) { + return span(reinterpret_cast(v.data()), v.size()); +} + +// ============================================================================= +// Status and Error codes +// ============================================================================= +enum class Error { + OK = 0, + // JSON parsing errors - json_parser.{h,cc}. + JSON_PARSER_UNPROCESSED_INPUT_REMAINS = 0x01, + JSON_PARSER_STACK_LIMIT_EXCEEDED = 0x02, + JSON_PARSER_NO_INPUT = 0x03, + JSON_PARSER_INVALID_TOKEN = 0x04, + JSON_PARSER_INVALID_NUMBER = 0x05, + JSON_PARSER_INVALID_STRING = 0x06, + JSON_PARSER_UNEXPECTED_ARRAY_END = 0x07, + JSON_PARSER_COMMA_OR_ARRAY_END_EXPECTED = 0x08, + JSON_PARSER_STRING_LITERAL_EXPECTED = 0x09, + JSON_PARSER_COLON_EXPECTED = 0x0a, + JSON_PARSER_UNEXPECTED_MAP_END = 0x0b, + JSON_PARSER_COMMA_OR_MAP_END_EXPECTED = 0x0c, + JSON_PARSER_VALUE_EXPECTED = 0x0d, + + CBOR_INVALID_INT32 = 0x0e, + CBOR_INVALID_DOUBLE = 0x0f, + CBOR_INVALID_ENVELOPE = 0x10, + CBOR_INVALID_STRING8 = 0x11, + CBOR_INVALID_STRING16 = 0x12, + CBOR_INVALID_BINARY = 0x13, + CBOR_UNSUPPORTED_VALUE = 0x14, + CBOR_NO_INPUT = 0x15, + CBOR_INVALID_START_BYTE = 0x16, + CBOR_UNEXPECTED_EOF_EXPECTED_VALUE = 0x17, + CBOR_UNEXPECTED_EOF_IN_ARRAY = 0x18, + CBOR_UNEXPECTED_EOF_IN_MAP = 0x19, + CBOR_INVALID_MAP_KEY = 0x1a, + CBOR_STACK_LIMIT_EXCEEDED = 0x1b, + CBOR_TRAILING_JUNK = 0x1c, + CBOR_MAP_START_EXPECTED = 0x1d, + CBOR_MAP_STOP_EXPECTED = 0x1e, + CBOR_ENVELOPE_SIZE_LIMIT_EXCEEDED = 0x1f, +}; + +// A status value with position that can be copied. The default status +// is OK. Usually, error status values should come with a valid position. +struct Status { + static constexpr size_t npos() { return std::numeric_limits::max(); } + + bool ok() const { return error == Error::OK; } + + Error error = Error::OK; + size_t pos = npos(); + Status(Error error, size_t pos) : error(error), pos(pos) {} + Status() = default; + + // Returns a 7 bit US-ASCII string, either "OK" or an error message + // that includes the position. + std::string ToASCIIString() const; + + private: + std::string ToASCIIString(const char* msg) const; +}; + +// Handler interface for parser events emitted by a streaming parser. +// See cbor::NewCBOREncoder, cbor::ParseCBOR, json::NewJSONEncoder, +// json::ParseJSON. +class StreamingParserHandler { + public: + virtual ~StreamingParserHandler() = default; + virtual void HandleMapBegin() = 0; + virtual void HandleMapEnd() = 0; + virtual void HandleArrayBegin() = 0; + virtual void HandleArrayEnd() = 0; + virtual void HandleString8(span chars) = 0; + virtual void HandleString16(span chars) = 0; + virtual void HandleBinary(span bytes) = 0; + virtual void HandleDouble(double value) = 0; + virtual void HandleInt32(int32_t value) = 0; + virtual void HandleBool(bool value) = 0; + virtual void HandleNull() = 0; + + // The parser may send one error even after other events have already + // been received. Client code is reponsible to then discard the + // already processed events. + // |error| must be an eror, as in, |error.is_ok()| can't be true. + virtual void HandleError(Status error) = 0; +}; + +namespace cbor { +// The binary encoding for the inspector protocol follows the CBOR specification +// (RFC 7049). Additional constraints: +// - Only indefinite length maps and arrays are supported. +// - Maps and arrays are wrapped with an envelope, that is, a +// CBOR tag with value 24 followed by a byte string specifying +// the byte length of the enclosed map / array. The byte string +// must use a 32 bit wide length. +// - At the top level, a message must be an indefinite length map +// wrapped by an envelope. +// - Maximal size for messages is 2^32 (4 GB). +// - For scalars, we support only the int32_t range, encoded as +// UNSIGNED/NEGATIVE (major types 0 / 1). +// - UTF16 strings, including with unbalanced surrogate pairs, are encoded +// as CBOR BYTE_STRING (major type 2). For such strings, the number of +// bytes encoded must be even. +// - UTF8 strings (major type 3) are supported. +// - 7 bit US-ASCII strings must always be encoded as UTF8 strings, never +// as UTF16 strings. +// - Arbitrary byte arrays, in the inspector protocol called 'binary', +// are encoded as BYTE_STRING (major type 2), prefixed with a byte +// indicating base64 when rendered as JSON. + +// ============================================================================= +// Detecting CBOR content +// ============================================================================= + +// The first byte for an envelope, which we use for wrapping dictionaries +// and arrays; and the byte that indicates a byte string with 32 bit length. +// These two bytes start an envelope, and thereby also any CBOR message +// produced or consumed by this protocol. See also |EnvelopeEncoder| below. +uint8_t InitialByteForEnvelope(); +uint8_t InitialByteFor32BitLengthByteString(); + +// Checks whether |msg| is a cbor message. +bool IsCBORMessage(span msg); + +// ============================================================================= +// Encoding individual CBOR items +// ============================================================================= + +// Some constants for CBOR tokens that only take a single byte on the wire. +uint8_t EncodeTrue(); +uint8_t EncodeFalse(); +uint8_t EncodeNull(); +uint8_t EncodeIndefiniteLengthArrayStart(); +uint8_t EncodeIndefiniteLengthMapStart(); +uint8_t EncodeStop(); + +// Encodes |value| as |UNSIGNED| (major type 0) iff >= 0, or |NEGATIVE| +// (major type 1) iff < 0. +void EncodeInt32(int32_t value, std::vector* out); +void EncodeInt32(int32_t value, std::string* out); + +// Encodes a UTF16 string as a BYTE_STRING (major type 2). Each utf16 +// character in |in| is emitted with most significant byte first, +// appending to |out|. +void EncodeString16(span in, std::vector* out); +void EncodeString16(span in, std::string* out); + +// Encodes a UTF8 string |in| as STRING (major type 3). +void EncodeString8(span in, std::vector* out); +void EncodeString8(span in, std::string* out); + +// Encodes the given |latin1| string as STRING8. +// If any non-ASCII character is present, it will be represented +// as a 2 byte UTF8 sequence. +void EncodeFromLatin1(span latin1, std::vector* out); +void EncodeFromLatin1(span latin1, std::string* out); + +// Encodes the given |utf16| string as STRING8 if it's entirely US-ASCII. +// Otherwise, encodes as STRING16. +void EncodeFromUTF16(span utf16, std::vector* out); +void EncodeFromUTF16(span utf16, std::string* out); + +// Encodes arbitrary binary data in |in| as a BYTE_STRING (major type 2) with +// definitive length, prefixed with tag 22 indicating expected conversion to +// base64 (see RFC 7049, Table 3 and Section 2.4.4.2). +void EncodeBinary(span in, std::vector* out); +void EncodeBinary(span in, std::string* out); + +// Encodes / decodes a double as Major type 7 (SIMPLE_VALUE), +// with additional info = 27, followed by 8 bytes in big endian. +void EncodeDouble(double value, std::vector* out); +void EncodeDouble(double value, std::string* out); + +// ============================================================================= +// cbor::EnvelopeEncoder - for wrapping submessages +// ============================================================================= + +// An envelope indicates the byte length of a wrapped item. +// We use this for maps and array, which allows the decoder +// to skip such (nested) values whole sale. +// It's implemented as a CBOR tag (major type 6) with additional +// info = 24, followed by a byte string with a 32 bit length value; +// so the maximal structure that we can wrap is 2^32 bits long. +// See also: https://tools.ietf.org/html/rfc7049#section-2.4.4.1 +class EnvelopeEncoder { + public: + // Emits the envelope start bytes and records the position for the + // byte size in |byte_size_pos_|. Also emits empty bytes for the + // byte sisze so that encoding can continue. + void EncodeStart(std::vector* out); + void EncodeStart(std::string* out); + // This records the current size in |out| at position byte_size_pos_. + // Returns true iff successful. + bool EncodeStop(std::vector* out); + bool EncodeStop(std::string* out); + + private: + size_t byte_size_pos_ = 0; +}; + +// ============================================================================= +// cbor::NewCBOREncoder - for encoding from a streaming parser +// ============================================================================= + +// This can be used to convert to CBOR, by passing the return value to a parser +// that drives it. The handler will encode into |out|, and iff an error occurs +// it will set |status| to an error and clear |out|. Otherwise, |status.ok()| +// will be |true|. +std::unique_ptr NewCBOREncoder( + std::vector* out, + Status* status); +std::unique_ptr NewCBOREncoder(std::string* out, + Status* status); + +// ============================================================================= +// cbor::CBORTokenizer - for parsing individual CBOR items +// ============================================================================= + +// Tags for the tokens within a CBOR message that CBORTokenizer understands. +// Note that this is not the same terminology as the CBOR spec (RFC 7049), +// but rather, our adaptation. For instance, we lump unsigned and signed +// major type into INT32 here (and disallow values outside the int32_t range). +enum class CBORTokenTag { + // Encountered an error in the structure of the message. Consult + // status() for details. + ERROR_VALUE, + // Booleans and NULL. + TRUE_VALUE, + FALSE_VALUE, + NULL_VALUE, + // An int32_t (signed 32 bit integer). + INT32, + // A double (64 bit floating point). + DOUBLE, + // A UTF8 string. + STRING8, + // A UTF16 string. + STRING16, + // A binary string. + BINARY, + // Starts an indefinite length map; after the map start we expect + // alternating keys and values, followed by STOP. + MAP_START, + // Starts an indefinite length array; after the array start we + // expect values, followed by STOP. + ARRAY_START, + // Ends a map or an array. + STOP, + // An envelope indicator, wrapping a map or array. + // Internally this carries the byte length of the wrapped + // map or array. While CBORTokenizer::Next() will read / skip the entire + // envelope, CBORTokenizer::EnterEnvelope() reads the tokens + // inside of it. + ENVELOPE, + // We've reached the end there is nothing else to read. + DONE, +}; + +// The major types from RFC 7049 Section 2.1. +enum class MajorType { + UNSIGNED = 0, + NEGATIVE = 1, + BYTE_STRING = 2, + STRING = 3, + ARRAY = 4, + MAP = 5, + TAG = 6, + SIMPLE_VALUE = 7 +}; + +// CBORTokenizer segments a CBOR message, presenting the tokens therein as +// numbers, strings, etc. This is not a complete CBOR parser, but makes it much +// easier to implement one (e.g. ParseCBOR, above). It can also be used to parse +// messages partially. +class CBORTokenizer { + public: + explicit CBORTokenizer(span bytes); + ~CBORTokenizer(); + + // Identifies the current token that we're looking at, + // or ERROR_VALUE (in which ase ::Status() has details) + // or DONE (if we're past the last token). + CBORTokenTag TokenTag() const; + + // Advances to the next token. + void Next(); + // Can only be called if TokenTag() == CBORTokenTag::ENVELOPE. + // While Next() would skip past the entire envelope / what it's + // wrapping, EnterEnvelope positions the cursor inside of the envelope, + // letting the client explore the nested structure. + void EnterEnvelope(); + + // If TokenTag() is CBORTokenTag::ERROR_VALUE, then Status().error describes + // the error more precisely; otherwise it'll be set to Error::OK. + // In either case, Status().pos is the current position. + struct Status Status() const; + + // The following methods retrieve the token values. They can only + // be called if TokenTag() matches. + + // To be called only if ::TokenTag() == CBORTokenTag::INT32. + int32_t GetInt32() const; + + // To be called only if ::TokenTag() == CBORTokenTag::DOUBLE. + double GetDouble() const; + + // To be called only if ::TokenTag() == CBORTokenTag::STRING8. + span GetString8() const; + + // Wire representation for STRING16 is low byte first (little endian). + // To be called only if ::TokenTag() == CBORTokenTag::STRING16. + span GetString16WireRep() const; + + // To be called only if ::TokenTag() == CBORTokenTag::BINARY. + span GetBinary() const; + + // To be called only if ::TokenTag() == CBORTokenTag::ENVELOPE. + span GetEnvelopeContents() const; + + private: + void ReadNextToken(bool enter_envelope); + void SetToken(CBORTokenTag token, size_t token_byte_length); + void SetError(Error error); + + span bytes_; + CBORTokenTag token_tag_; + struct Status status_; + size_t token_byte_length_; + MajorType token_start_type_; + uint64_t token_start_internal_value_; +}; + +// ============================================================================= +// cbor::ParseCBOR - for receiving streaming parser events for CBOR messages +// ============================================================================= + +// Parses a CBOR encoded message from |bytes|, sending events to +// |out|. If an error occurs, sends |out->HandleError|, and parsing stops. +// The client is responsible for discarding the already received information in +// that case. +void ParseCBOR(span bytes, StreamingParserHandler* out); + +// ============================================================================= +// cbor::AppendString8EntryToMap - for limited in-place editing of messages +// ============================================================================= + +// Modifies the |cbor| message by appending a new key/value entry at the end +// of the map. Patches up the envelope size; Status.ok() iff successful. +// If not successful, |cbor| may be corrupted after this call. +Status AppendString8EntryToCBORMap(span string8_key, + span string8_value, + std::vector* cbor); +Status AppendString8EntryToCBORMap(span string8_key, + span string8_value, + std::string* cbor); + +namespace internals { // Exposed only for writing tests. +int8_t ReadTokenStart(span bytes, + cbor::MajorType* type, + uint64_t* value); + +void WriteTokenStart(cbor::MajorType type, + uint64_t value, + std::vector* encoded); +void WriteTokenStart(cbor::MajorType type, + uint64_t value, + std::string* encoded); +} // namespace internals +} // namespace cbor + +namespace json { +// Client code must provide an instance. Implementation should delegate +// to whatever is appropriate. +class Platform { + public: + virtual ~Platform() = default; + // Parses |str| into |result|. Returns false iff there are + // leftover characters or parsing errors. + virtual bool StrToD(const char* str, double* result) const = 0; + + // Prints |value| in a format suitable for JSON. + virtual std::unique_ptr DToStr(double value) const = 0; +}; + +// ============================================================================= +// json::NewJSONEncoder - for encoding streaming parser events as JSON +// ============================================================================= + +// Returns a handler object which will write ascii characters to |out|. +// |status->ok()| will be false iff the handler routine HandleError() is called. +// In that case, we'll stop emitting output. +// Except for calling the HandleError routine at any time, the client +// code must call the Handle* methods in an order in which they'd occur +// in valid JSON; otherwise we may crash (the code uses assert). +std::unique_ptr NewJSONEncoder( + const Platform* platform, + std::vector* out, + Status* status); +std::unique_ptr NewJSONEncoder(const Platform* platform, + std::string* out, + Status* status); + +// ============================================================================= +// json::ParseJSON - for receiving streaming parser events for JSON +// ============================================================================= + +void ParseJSON(const Platform& platform, + span chars, + StreamingParserHandler* handler); +void ParseJSON(const Platform& platform, + span chars, + StreamingParserHandler* handler); + +// ============================================================================= +// json::ConvertCBORToJSON, json::ConvertJSONToCBOR - for transcoding +// ============================================================================= +Status ConvertCBORToJSON(const Platform& platform, + span cbor, + std::string* json); +Status ConvertCBORToJSON(const Platform& platform, + span cbor, + std::vector* json); +Status ConvertJSONToCBOR(const Platform& platform, + span json, + std::vector* cbor); +Status ConvertJSONToCBOR(const Platform& platform, + span json, + std::vector* cbor); +Status ConvertJSONToCBOR(const Platform& platform, + span json, + std::string* cbor); +Status ConvertJSONToCBOR(const Platform& platform, + span json, + std::string* cbor); +} // namespace json +} // namespace v8_inspector_protocol_encoding + +#endif // V8_INSPECTOR_PROTOCOL_ENCODING_ENCODING_H_ diff --git a/tools/inspector_protocol/encoding/encoding_test.cc b/tools/inspector_protocol/encoding/encoding_test.cc new file mode 100644 index 00000000000000..b8d75e09baaf31 --- /dev/null +++ b/tools/inspector_protocol/encoding/encoding_test.cc @@ -0,0 +1,1838 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "encoding.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "encoding_test_helper.h" + +using testing::ElementsAreArray; + +namespace v8_inspector_protocol_encoding { + +class TestPlatform : public json::Platform { + bool StrToD(const char* str, double* result) const override { + // This is not thread-safe + // (see https://en.cppreference.com/w/cpp/locale/setlocale) + // but good enough for a unittest. + const char* saved_locale = std::setlocale(LC_NUMERIC, nullptr); + char* end; + *result = std::strtod(str, &end); + std::setlocale(LC_NUMERIC, saved_locale); + if (errno == ERANGE) { + // errno must be reset, e.g. see the example here: + // https://en.cppreference.com/w/cpp/string/byte/strtof + errno = 0; + return false; + } + return end == str + strlen(str); + } + + std::unique_ptr DToStr(double value) const override { + std::stringstream ss; + ss.imbue(std::locale("C")); + ss << value; + std::string str = ss.str(); + std::unique_ptr result(new char[str.size() + 1]); + memcpy(result.get(), str.c_str(), str.size() + 1); + return result; + } +}; + +const json::Platform& GetTestPlatform() { + static TestPlatform* platform = new TestPlatform; + return *platform; +} + +// ============================================================================= +// span - sequence of bytes +// ============================================================================= + +template +class SpanTest : public ::testing::Test {}; + +using TestTypes = ::testing::Types; +TYPED_TEST_SUITE(SpanTest, TestTypes); + +TYPED_TEST(SpanTest, Empty) { + span empty; + EXPECT_TRUE(empty.empty()); + EXPECT_EQ(0u, empty.size()); + EXPECT_EQ(0u, empty.size_bytes()); + EXPECT_EQ(empty.begin(), empty.end()); +} + +TYPED_TEST(SpanTest, SingleItem) { + TypeParam single_item = 42; + span singular(&single_item, 1); + EXPECT_FALSE(singular.empty()); + EXPECT_EQ(1u, singular.size()); + EXPECT_EQ(sizeof(TypeParam), singular.size_bytes()); + EXPECT_EQ(singular.begin() + 1, singular.end()); + EXPECT_EQ(42, singular[0]); +} + +TYPED_TEST(SpanTest, FiveItems) { + std::vector test_input = {31, 32, 33, 34, 35}; + span five_items(test_input.data(), 5); + EXPECT_FALSE(five_items.empty()); + EXPECT_EQ(5u, five_items.size()); + EXPECT_EQ(sizeof(TypeParam) * 5, five_items.size_bytes()); + EXPECT_EQ(five_items.begin() + 5, five_items.end()); + EXPECT_EQ(31, five_items[0]); + EXPECT_EQ(32, five_items[1]); + EXPECT_EQ(33, five_items[2]); + EXPECT_EQ(34, five_items[3]); + EXPECT_EQ(35, five_items[4]); + span three_items = five_items.subspan(2); + EXPECT_EQ(3u, three_items.size()); + EXPECT_EQ(33, three_items[0]); + EXPECT_EQ(34, three_items[1]); + EXPECT_EQ(35, three_items[2]); + span two_items = five_items.subspan(2, 2); + EXPECT_EQ(2u, two_items.size()); + EXPECT_EQ(33, two_items[0]); + EXPECT_EQ(34, two_items[1]); +} + +TEST(SpanFromTest, FromConstCharAndLiteral) { + // Testing this is useful because strlen(nullptr) is undefined. + EXPECT_EQ(nullptr, SpanFrom(nullptr).data()); + EXPECT_EQ(0u, SpanFrom(nullptr).size()); + + const char* kEmpty = ""; + EXPECT_EQ(kEmpty, reinterpret_cast(SpanFrom(kEmpty).data())); + EXPECT_EQ(0u, SpanFrom(kEmpty).size()); + + const char* kFoo = "foo"; + EXPECT_EQ(kFoo, reinterpret_cast(SpanFrom(kFoo).data())); + EXPECT_EQ(3u, SpanFrom(kFoo).size()); + + EXPECT_EQ(3u, SpanFrom("foo").size()); +} + +// ============================================================================= +// Status and Error codes +// ============================================================================= + +TEST(StatusTest, StatusToASCIIString) { + Status ok_status; + EXPECT_EQ("OK", ok_status.ToASCIIString()); + Status json_error(Error::JSON_PARSER_COLON_EXPECTED, 42); + EXPECT_EQ("JSON: colon expected at position 42", json_error.ToASCIIString()); + Status cbor_error(Error::CBOR_TRAILING_JUNK, 21); + EXPECT_EQ("CBOR: trailing junk at position 21", cbor_error.ToASCIIString()); +} + +namespace cbor { + +// ============================================================================= +// Detecting CBOR content +// ============================================================================= + +TEST(IsCBORMessage, SomeSmokeTests) { + std::vector empty; + EXPECT_FALSE(IsCBORMessage(SpanFrom(empty))); + std::vector hello = {'H', 'e', 'l', 'o', ' ', 't', + 'h', 'e', 'r', 'e', '!'}; + EXPECT_FALSE(IsCBORMessage(SpanFrom(hello))); + std::vector example = {0xd8, 0x5a, 0, 0, 0, 0}; + EXPECT_TRUE(IsCBORMessage(SpanFrom(example))); + std::vector one = {0xd8, 0x5a, 0, 0, 0, 1, 1}; + EXPECT_TRUE(IsCBORMessage(SpanFrom(one))); +} + +// ============================================================================= +// Encoding individual CBOR items +// cbor::CBORTokenizer - for parsing individual CBOR items +// ============================================================================= + +// +// EncodeInt32 / CBORTokenTag::INT32 +// +TEST(EncodeDecodeInt32Test, Roundtrips23) { + // This roundtrips the int32_t value 23 through the pair of EncodeInt32 / + // CBORTokenizer; this is interesting since 23 is encoded as a single byte. + std::vector encoded; + EncodeInt32(23, &encoded); + // first three bits: major type = 0; remaining five bits: additional info = + // value 23. + EXPECT_THAT(encoded, ElementsAreArray(std::array{{23}})); + + // Reverse direction: decode with CBORTokenizer. + CBORTokenizer tokenizer(SpanFrom(encoded)); + EXPECT_EQ(CBORTokenTag::INT32, tokenizer.TokenTag()); + EXPECT_EQ(23, tokenizer.GetInt32()); + tokenizer.Next(); + EXPECT_EQ(CBORTokenTag::DONE, tokenizer.TokenTag()); +} + +TEST(EncodeDecodeInt32Test, RoundtripsUint8) { + // This roundtrips the int32_t value 42 through the pair of EncodeInt32 / + // CBORTokenizer. This is different from Roundtrip23 because 42 is encoded + // in an extra byte after the initial one. + std::vector encoded; + EncodeInt32(42, &encoded); + // first three bits: major type = 0; + // remaining five bits: additional info = 24, indicating payload is uint8. + EXPECT_THAT(encoded, ElementsAreArray(std::array{{24, 42}})); + + // Reverse direction: decode with CBORTokenizer. + CBORTokenizer tokenizer(SpanFrom(encoded)); + EXPECT_EQ(CBORTokenTag::INT32, tokenizer.TokenTag()); + EXPECT_EQ(42, tokenizer.GetInt32()); + tokenizer.Next(); + EXPECT_EQ(CBORTokenTag::DONE, tokenizer.TokenTag()); +} + +TEST(EncodeDecodeInt32Test, RoundtripsUint16) { + // 500 is encoded as a uint16 after the initial byte. + std::vector encoded; + EncodeInt32(500, &encoded); + // 1 for initial byte, 2 for uint16. + EXPECT_EQ(3u, encoded.size()); + // first three bits: major type = 0; + // remaining five bits: additional info = 25, indicating payload is uint16. + EXPECT_EQ(25, encoded[0]); + EXPECT_EQ(0x01, encoded[1]); + EXPECT_EQ(0xf4, encoded[2]); + + // Reverse direction: decode with CBORTokenizer. + CBORTokenizer tokenizer(SpanFrom(encoded)); + EXPECT_EQ(CBORTokenTag::INT32, tokenizer.TokenTag()); + EXPECT_EQ(500, tokenizer.GetInt32()); + tokenizer.Next(); + EXPECT_EQ(CBORTokenTag::DONE, tokenizer.TokenTag()); +} + +TEST(EncodeDecodeInt32Test, RoundtripsInt32Max) { + // std::numeric_limits is encoded as a uint32 after the initial byte. + std::vector encoded; + EncodeInt32(std::numeric_limits::max(), &encoded); + // 1 for initial byte, 4 for the uint32. + // first three bits: major type = 0; + // remaining five bits: additional info = 26, indicating payload is uint32. + EXPECT_THAT( + encoded, + ElementsAreArray(std::array{{26, 0x7f, 0xff, 0xff, 0xff}})); + + // Reverse direction: decode with CBORTokenizer. + CBORTokenizer tokenizer(SpanFrom(encoded)); + EXPECT_EQ(CBORTokenTag::INT32, tokenizer.TokenTag()); + EXPECT_EQ(std::numeric_limits::max(), tokenizer.GetInt32()); + tokenizer.Next(); + EXPECT_EQ(CBORTokenTag::DONE, tokenizer.TokenTag()); +} + +TEST(EncodeDecodeInt32Test, CantRoundtripUint32) { + // 0xdeadbeef is a value which does not fit below + // std::numerical_limits::max(), so we can't encode + // it with EncodeInt32. However, CBOR does support this, so we + // encode it here manually with the internal routine, just to observe + // that it's considered an invalid int32 by CBORTokenizer. + std::vector encoded; + internals::WriteTokenStart(MajorType::UNSIGNED, 0xdeadbeef, &encoded); + // 1 for initial byte, 4 for the uint32. + // first three bits: major type = 0; + // remaining five bits: additional info = 26, indicating payload is uint32. + EXPECT_THAT( + encoded, + ElementsAreArray(std::array{{26, 0xde, 0xad, 0xbe, 0xef}})); + + // Now try to decode; we treat this as an invalid INT32. + CBORTokenizer tokenizer(SpanFrom(encoded)); + // 0xdeadbeef is > std::numerical_limits::max(). + EXPECT_EQ(CBORTokenTag::ERROR_VALUE, tokenizer.TokenTag()); + EXPECT_EQ(Error::CBOR_INVALID_INT32, tokenizer.Status().error); +} + +TEST(EncodeDecodeInt32Test, DecodeErrorCases) { + struct TestCase { + std::vector data; + std::string msg; + }; + std::vector tests{ + {TestCase{ + {24}, + "additional info = 24 would require 1 byte of payload (but it's 0)"}, + TestCase{{27, 0xaa, 0xbb, 0xcc}, + "additional info = 27 would require 8 bytes of payload (but " + "it's 3)"}, + TestCase{{29}, "additional info = 29 isn't recognized"}}}; + + for (const TestCase& test : tests) { + SCOPED_TRACE(test.msg); + CBORTokenizer tokenizer(SpanFrom(test.data)); + EXPECT_EQ(CBORTokenTag::ERROR_VALUE, tokenizer.TokenTag()); + EXPECT_EQ(Error::CBOR_INVALID_INT32, tokenizer.Status().error); + } +} + +TEST(EncodeDecodeInt32Test, RoundtripsMinus24) { + // This roundtrips the int32_t value -24 through the pair of EncodeInt32 / + // CBORTokenizer; this is interesting since -24 is encoded as + // a single byte as NEGATIVE, and it tests the specific encoding + // (note how for unsigned the single byte covers values up to 23). + // Additional examples are covered in RoundtripsAdditionalExamples. + std::vector encoded; + EncodeInt32(-24, &encoded); + // first three bits: major type = 1; remaining five bits: additional info = + // value 23. + EXPECT_THAT(encoded, ElementsAreArray(std::array{{1 << 5 | 23}})); + + // Reverse direction: decode with CBORTokenizer. + CBORTokenizer tokenizer(SpanFrom(encoded)); + EXPECT_EQ(CBORTokenTag::INT32, tokenizer.TokenTag()); + EXPECT_EQ(-24, tokenizer.GetInt32()); + tokenizer.Next(); + EXPECT_EQ(CBORTokenTag::DONE, tokenizer.TokenTag()); +} + +TEST(EncodeDecodeInt32Test, RoundtripsAdditionalNegativeExamples) { + std::vector examples = {-1, + -10, + -24, + -25, + -300, + -30000, + -300 * 1000, + -1000 * 1000, + -1000 * 1000 * 1000, + std::numeric_limits::min()}; + for (int32_t example : examples) { + SCOPED_TRACE(std::string("example ") + std::to_string(example)); + std::vector encoded; + EncodeInt32(example, &encoded); + CBORTokenizer tokenizer(SpanFrom(encoded)); + EXPECT_EQ(CBORTokenTag::INT32, tokenizer.TokenTag()); + EXPECT_EQ(example, tokenizer.GetInt32()); + tokenizer.Next(); + EXPECT_EQ(CBORTokenTag::DONE, tokenizer.TokenTag()); + } +} + +// +// EncodeString16 / CBORTokenTag::STRING16 +// +TEST(EncodeDecodeString16Test, RoundtripsEmpty) { + // This roundtrips the empty utf16 string through the pair of EncodeString16 / + // CBORTokenizer. + std::vector encoded; + EncodeString16(span(), &encoded); + EXPECT_EQ(1u, encoded.size()); + // first three bits: major type = 2; remaining five bits: additional info = + // size 0. + EXPECT_EQ(2 << 5, encoded[0]); + + // Reverse direction: decode with CBORTokenizer. + CBORTokenizer tokenizer(SpanFrom(encoded)); + EXPECT_EQ(CBORTokenTag::STRING16, tokenizer.TokenTag()); + span decoded_string16_wirerep = tokenizer.GetString16WireRep(); + EXPECT_TRUE(decoded_string16_wirerep.empty()); + tokenizer.Next(); + EXPECT_EQ(CBORTokenTag::DONE, tokenizer.TokenTag()); +} + +// On the wire, we STRING16 is encoded as little endian (least +// significant byte first). The host may or may not be little endian, +// so this routine follows the advice in +// https://commandcenter.blogspot.com/2012/04/byte-order-fallacy.html. +std::vector String16WireRepToHost(span in) { + // must be even number of bytes. + CHECK_EQ(in.size() & 1, 0u); + std::vector host_out; + for (size_t ii = 0; ii < in.size(); ii += 2) + host_out.push_back(in[ii + 1] << 8 | in[ii]); + return host_out; +} + +TEST(EncodeDecodeString16Test, RoundtripsHelloWorld) { + // This roundtrips the hello world message which is given here in utf16 + // characters. 0xd83c, 0xdf0e: UTF16 encoding for the "Earth Globe Americas" + // character, 🌎. + std::array msg{ + {'H', 'e', 'l', 'l', 'o', ',', ' ', 0xd83c, 0xdf0e, '.'}}; + std::vector encoded; + EncodeString16(span(msg.data(), msg.size()), &encoded); + // This will be encoded as BYTE_STRING of length 20, so the 20 is encoded in + // the additional info part of the initial byte. Payload is two bytes for each + // UTF16 character. + uint8_t initial_byte = /*major type=*/2 << 5 | /*additional info=*/20; + std::array encoded_expected = { + {initial_byte, 'H', 0, 'e', 0, 'l', 0, 'l', 0, 'o', 0, + ',', 0, ' ', 0, 0x3c, 0xd8, 0x0e, 0xdf, '.', 0}}; + EXPECT_THAT(encoded, ElementsAreArray(encoded_expected)); + + // Now decode to complete the roundtrip. + CBORTokenizer tokenizer(SpanFrom(encoded)); + EXPECT_EQ(CBORTokenTag::STRING16, tokenizer.TokenTag()); + std::vector decoded = + String16WireRepToHost(tokenizer.GetString16WireRep()); + EXPECT_THAT(decoded, ElementsAreArray(msg)); + tokenizer.Next(); + EXPECT_EQ(CBORTokenTag::DONE, tokenizer.TokenTag()); + + // For bonus points, we look at the decoded message in UTF8 as well so we can + // easily see it on the terminal screen. + std::string utf8_decoded = UTF16ToUTF8(SpanFrom(decoded)); + EXPECT_EQ("Hello, 🌎.", utf8_decoded); +} + +TEST(EncodeDecodeString16Test, Roundtrips500) { + // We roundtrip a message that has 250 16 bit values. Each of these are just + // set to their index. 250 is interesting because the cbor spec uses a + // BYTE_STRING of length 500 for one of their examples of how to encode the + // start of it (section 2.1) so it's easy for us to look at the first three + // bytes closely. + std::vector two_fifty; + for (uint16_t ii = 0; ii < 250; ++ii) + two_fifty.push_back(ii); + std::vector encoded; + EncodeString16(span(two_fifty.data(), two_fifty.size()), &encoded); + EXPECT_EQ(3u + 250u * 2, encoded.size()); + // Now check the first three bytes: + // Major type: 2 (BYTE_STRING) + // Additional information: 25, indicating size is represented by 2 bytes. + // Bytes 1 and 2 encode 500 (0x01f4). + EXPECT_EQ(2 << 5 | 25, encoded[0]); + EXPECT_EQ(0x01, encoded[1]); + EXPECT_EQ(0xf4, encoded[2]); + + // Now decode to complete the roundtrip. + CBORTokenizer tokenizer(SpanFrom(encoded)); + EXPECT_EQ(CBORTokenTag::STRING16, tokenizer.TokenTag()); + std::vector decoded = + String16WireRepToHost(tokenizer.GetString16WireRep()); + EXPECT_THAT(decoded, ElementsAreArray(two_fifty)); + tokenizer.Next(); + EXPECT_EQ(CBORTokenTag::DONE, tokenizer.TokenTag()); +} + +TEST(EncodeDecodeString16Test, ErrorCases) { + struct TestCase { + std::vector data; + std::string msg; + }; + std::vector tests{ + {TestCase{{2 << 5 | 1, 'a'}, + "length must be divisible by 2 (but it's 1)"}, + TestCase{{2 << 5 | 29}, "additional info = 29 isn't recognized"}, + TestCase{{2 << 5 | 9, 1, 2, 3, 4, 5, 6, 7, 8}, + "length (9) points just past the end of the test case"}, + TestCase{{2 << 5 | 27, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 'a', 'b', 'c'}, + "large length pointing past the end of the test case"}}}; + for (const TestCase& test : tests) { + SCOPED_TRACE(test.msg); + CBORTokenizer tokenizer(SpanFrom(test.data)); + EXPECT_EQ(CBORTokenTag::ERROR_VALUE, tokenizer.TokenTag()); + EXPECT_EQ(Error::CBOR_INVALID_STRING16, tokenizer.Status().error); + } +} + +// +// EncodeString8 / CBORTokenTag::STRING8 +// +TEST(EncodeDecodeString8Test, RoundtripsHelloWorld) { + // This roundtrips the hello world message which is given here in utf8 + // characters. 🌎 is a four byte utf8 character. + std::string utf8_msg = "Hello, 🌎."; + std::vector msg(utf8_msg.begin(), utf8_msg.end()); + std::vector encoded; + EncodeString8(SpanFrom(utf8_msg), &encoded); + // This will be encoded as STRING of length 12, so the 12 is encoded in + // the additional info part of the initial byte. Payload is one byte per + // utf8 byte. + uint8_t initial_byte = /*major type=*/3 << 5 | /*additional info=*/12; + std::array encoded_expected = {{initial_byte, 'H', 'e', 'l', 'l', + 'o', ',', ' ', 0xF0, 0x9f, 0x8c, + 0x8e, '.'}}; + EXPECT_THAT(encoded, ElementsAreArray(encoded_expected)); + + // Now decode to complete the roundtrip. + CBORTokenizer tokenizer(SpanFrom(encoded)); + EXPECT_EQ(CBORTokenTag::STRING8, tokenizer.TokenTag()); + std::vector decoded(tokenizer.GetString8().begin(), + tokenizer.GetString8().end()); + EXPECT_THAT(decoded, ElementsAreArray(msg)); + tokenizer.Next(); + EXPECT_EQ(CBORTokenTag::DONE, tokenizer.TokenTag()); +} + +TEST(EncodeDecodeString8Test, ErrorCases) { + struct TestCase { + std::vector data; + std::string msg; + }; + std::vector tests{ + {TestCase{{3 << 5 | 29}, "additional info = 29 isn't recognized"}, + TestCase{{3 << 5 | 9, 1, 2, 3, 4, 5, 6, 7, 8}, + "length (9) points just past the end of the test case"}, + TestCase{{3 << 5 | 27, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 'a', 'b', 'c'}, + "large length pointing past the end of the test case"}}}; + for (const TestCase& test : tests) { + SCOPED_TRACE(test.msg); + CBORTokenizer tokenizer(SpanFrom(test.data)); + EXPECT_EQ(CBORTokenTag::ERROR_VALUE, tokenizer.TokenTag()); + EXPECT_EQ(Error::CBOR_INVALID_STRING8, tokenizer.Status().error); + } +} + +TEST(EncodeFromLatin1Test, ConvertsToUTF8IfNeeded) { + std::vector> examples = { + {"Hello, world.", "Hello, world."}, + {"Above: \xDC" + "ber", + "Above: Über"}, + {"\xA5 500 are about \xA3 3.50; a y with umlaut is \xFF", + "¥ 500 are about £ 3.50; a y with umlaut is ÿ"}}; + + for (const auto& example : examples) { + const std::string& latin1 = example.first; + const std::string& expected_utf8 = example.second; + std::vector encoded; + EncodeFromLatin1(SpanFrom(latin1), &encoded); + CBORTokenizer tokenizer(SpanFrom(encoded)); + EXPECT_EQ(CBORTokenTag::STRING8, tokenizer.TokenTag()); + std::vector decoded(tokenizer.GetString8().begin(), + tokenizer.GetString8().end()); + std::string decoded_str(decoded.begin(), decoded.end()); + EXPECT_THAT(decoded_str, testing::Eq(expected_utf8)); + } +} + +TEST(EncodeFromUTF16Test, ConvertsToUTF8IfEasy) { + std::vector ascii = {'e', 'a', 's', 'y'}; + std::vector encoded; + EncodeFromUTF16(span(ascii.data(), ascii.size()), &encoded); + + CBORTokenizer tokenizer(SpanFrom(encoded)); + EXPECT_EQ(CBORTokenTag::STRING8, tokenizer.TokenTag()); + std::vector decoded(tokenizer.GetString8().begin(), + tokenizer.GetString8().end()); + std::string decoded_str(decoded.begin(), decoded.end()); + EXPECT_THAT(decoded_str, testing::Eq("easy")); +} + +TEST(EncodeFromUTF16Test, EncodesAsString16IfNeeded) { + // Since this message contains non-ASCII characters, the routine is + // forced to encode as UTF16. We see this below by checking that the + // token tag is STRING16. + std::vector msg = {'H', 'e', 'l', 'l', 'o', + ',', ' ', 0xd83c, 0xdf0e, '.'}; + std::vector encoded; + EncodeFromUTF16(span(msg.data(), msg.size()), &encoded); + + CBORTokenizer tokenizer(SpanFrom(encoded)); + EXPECT_EQ(CBORTokenTag::STRING16, tokenizer.TokenTag()); + std::vector decoded = + String16WireRepToHost(tokenizer.GetString16WireRep()); + std::string utf8_decoded = UTF16ToUTF8(SpanFrom(decoded)); + EXPECT_EQ("Hello, 🌎.", utf8_decoded); +} + +// +// EncodeBinary / CBORTokenTag::BINARY +// +TEST(EncodeDecodeBinaryTest, RoundtripsHelloWorld) { + std::vector binary = {'H', 'e', 'l', 'l', 'o', ',', ' ', + 'w', 'o', 'r', 'l', 'd', '.'}; + std::vector encoded; + EncodeBinary(span(binary.data(), binary.size()), &encoded); + // So, on the wire we see that the binary blob travels unmodified. + EXPECT_THAT( + encoded, + ElementsAreArray(std::array{ + {(6 << 5 | 22), // tag 22 indicating base64 interpretation in JSON + (2 << 5 | 13), // BYTE_STRING (type 2) of length 13 + 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '.'}})); + std::vector decoded; + CBORTokenizer tokenizer(SpanFrom(encoded)); + EXPECT_EQ(CBORTokenTag::BINARY, tokenizer.TokenTag()); + EXPECT_EQ(0, static_cast(tokenizer.Status().error)); + decoded = std::vector(tokenizer.GetBinary().begin(), + tokenizer.GetBinary().end()); + EXPECT_THAT(decoded, ElementsAreArray(binary)); + tokenizer.Next(); + EXPECT_EQ(CBORTokenTag::DONE, tokenizer.TokenTag()); +} + +TEST(EncodeDecodeBinaryTest, ErrorCases) { + struct TestCase { + std::vector data; + std::string msg; + }; + std::vector tests{{TestCase{ + {6 << 5 | 22, // tag 22 indicating base64 interpretation in JSON + 2 << 5 | 27, // BYTE_STRING (type 2), followed by 8 bytes length + 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + "large length pointing past the end of the test case"}}}; + for (const TestCase& test : tests) { + SCOPED_TRACE(test.msg); + CBORTokenizer tokenizer(SpanFrom(test.data)); + EXPECT_EQ(CBORTokenTag::ERROR_VALUE, tokenizer.TokenTag()); + EXPECT_EQ(Error::CBOR_INVALID_BINARY, tokenizer.Status().error); + } +} + +// +// EncodeDouble / CBORTokenTag::DOUBLE +// +TEST(EncodeDecodeDoubleTest, RoundtripsWikipediaExample) { + // https://en.wikipedia.org/wiki/Double-precision_floating-point_format + // provides the example of a hex representation 3FD5 5555 5555 5555, which + // approximates 1/3. + + const double kOriginalValue = 1.0 / 3; + std::vector encoded; + EncodeDouble(kOriginalValue, &encoded); + // first three bits: major type = 7; remaining five bits: additional info = + // value 27. This is followed by 8 bytes of payload (which match Wikipedia). + EXPECT_THAT( + encoded, + ElementsAreArray(std::array{ + {7 << 5 | 27, 0x3f, 0xd5, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55}})); + + // Reverse direction: decode and compare with original value. + CBORTokenizer tokenizer(SpanFrom(encoded)); + EXPECT_EQ(CBORTokenTag::DOUBLE, tokenizer.TokenTag()); + EXPECT_THAT(tokenizer.GetDouble(), testing::DoubleEq(kOriginalValue)); + tokenizer.Next(); + EXPECT_EQ(CBORTokenTag::DONE, tokenizer.TokenTag()); +} + +TEST(EncodeDecodeDoubleTest, RoundtripsAdditionalExamples) { + std::vector examples = {0.0, + 1.0, + -1.0, + 3.1415, + std::numeric_limits::min(), + std::numeric_limits::max(), + std::numeric_limits::infinity(), + std::numeric_limits::quiet_NaN()}; + for (double example : examples) { + SCOPED_TRACE(std::string("example ") + std::to_string(example)); + std::vector encoded; + EncodeDouble(example, &encoded); + CBORTokenizer tokenizer(SpanFrom(encoded)); + EXPECT_EQ(CBORTokenTag::DOUBLE, tokenizer.TokenTag()); + if (std::isnan(example)) + EXPECT_TRUE(std::isnan(tokenizer.GetDouble())); + else + EXPECT_THAT(tokenizer.GetDouble(), testing::DoubleEq(example)); + tokenizer.Next(); + EXPECT_EQ(CBORTokenTag::DONE, tokenizer.TokenTag()); + } +} + +// ============================================================================= +// cbor::NewCBOREncoder - for encoding from a streaming parser +// ============================================================================= + +void EncodeUTF8ForTest(const std::string& key, std::vector* out) { + EncodeString8(SpanFrom(key), out); +} +TEST(JSONToCBOREncoderTest, SevenBitStrings) { + // When a string can be represented as 7 bit ASCII, the encoder will use the + // STRING (major Type 3) type, so the actual characters end up as bytes on the + // wire. + std::vector encoded; + Status status; + std::unique_ptr encoder = + NewCBOREncoder(&encoded, &status); + std::vector utf16 = {'f', 'o', 'o'}; + encoder->HandleString16(span(utf16.data(), utf16.size())); + EXPECT_EQ(Error::OK, status.error); + // Here we assert that indeed, seven bit strings are represented as + // bytes on the wire, "foo" is just "foo". + EXPECT_THAT(encoded, + ElementsAreArray(std::array{ + {/*major type 3*/ 3 << 5 | /*length*/ 3, 'f', 'o', 'o'}})); +} + +TEST(JsonCborRoundtrip, EncodingDecoding) { + // Hits all the cases except binary and error in StreamingParserHandler, first + // parsing a JSON message into CBOR, then parsing it back from CBOR into JSON. + std::string json = + "{" + "\"string\":\"Hello, \\ud83c\\udf0e.\"," + "\"double\":3.1415," + "\"int\":1," + "\"negative int\":-1," + "\"bool\":true," + "\"null\":null," + "\"array\":[1,2,3]" + "}"; + std::vector encoded; + Status status; + std::unique_ptr encoder = + NewCBOREncoder(&encoded, &status); + span ascii_in = SpanFrom(json); + json::ParseJSON(GetTestPlatform(), ascii_in, encoder.get()); + std::vector expected = { + 0xd8, // envelope + 0x5a, // byte string with 32 bit length + 0, 0, 0, 94, // length is 94 bytes + }; + expected.push_back(0xbf); // indef length map start + EncodeString8(SpanFrom("string"), &expected); + // This is followed by the encoded string for "Hello, 🌎." + // So, it's the same bytes that we tested above in + // EncodeDecodeString16Test.RoundtripsHelloWorld. + expected.push_back(/*major type=*/2 << 5 | /*additional info=*/20); + for (uint8_t ch : std::array{ + {'H', 0, 'e', 0, 'l', 0, 'l', 0, 'o', 0, + ',', 0, ' ', 0, 0x3c, 0xd8, 0x0e, 0xdf, '.', 0}}) + expected.push_back(ch); + EncodeString8(SpanFrom("double"), &expected); + EncodeDouble(3.1415, &expected); + EncodeString8(SpanFrom("int"), &expected); + EncodeInt32(1, &expected); + EncodeString8(SpanFrom("negative int"), &expected); + EncodeInt32(-1, &expected); + EncodeString8(SpanFrom("bool"), &expected); + expected.push_back(7 << 5 | 21); // RFC 7049 Section 2.3, Table 2: true + EncodeString8(SpanFrom("null"), &expected); + expected.push_back(7 << 5 | 22); // RFC 7049 Section 2.3, Table 2: null + EncodeString8(SpanFrom("array"), &expected); + expected.push_back(0xd8); // envelope + expected.push_back(0x5a); // byte string with 32 bit length + // the length is 5 bytes (that's up to end indef length array below). + for (uint8_t ch : std::array{{0, 0, 0, 5}}) + expected.push_back(ch); + expected.push_back(0x9f); // RFC 7049 Section 2.2.1, indef length array start + expected.push_back(1); // Three UNSIGNED values (easy since Major Type 0) + expected.push_back(2); + expected.push_back(3); + expected.push_back(0xff); // End indef length array + expected.push_back(0xff); // End indef length map + EXPECT_TRUE(status.ok()); + EXPECT_THAT(encoded, ElementsAreArray(expected)); + + // And now we roundtrip, decoding the message we just encoded. + std::string decoded; + std::unique_ptr json_encoder = + NewJSONEncoder(&GetTestPlatform(), &decoded, &status); + ParseCBOR(span(encoded.data(), encoded.size()), json_encoder.get()); + EXPECT_EQ(Error::OK, status.error); + EXPECT_EQ(json, decoded); +} + +TEST(JsonCborRoundtrip, MoreRoundtripExamples) { + std::vector examples = { + // Tests that after closing a nested objects, additional key/value pairs + // are considered. + "{\"foo\":{\"bar\":1},\"baz\":2}", "{\"foo\":[1,2,3],\"baz\":2}"}; + for (const std::string& json : examples) { + SCOPED_TRACE(std::string("example: ") + json); + std::vector encoded; + Status status; + std::unique_ptr encoder = + NewCBOREncoder(&encoded, &status); + span ascii_in = SpanFrom(json); + ParseJSON(GetTestPlatform(), ascii_in, encoder.get()); + std::string decoded; + std::unique_ptr json_writer = + NewJSONEncoder(&GetTestPlatform(), &decoded, &status); + ParseCBOR(span(encoded.data(), encoded.size()), json_writer.get()); + EXPECT_EQ(Error::OK, status.error); + EXPECT_EQ(json, decoded); + } +} + +TEST(JSONToCBOREncoderTest, HelloWorldBinary_WithTripToJson) { + // The StreamingParserHandler::HandleBinary is a special case: The JSON parser + // will never call this method, because JSON does not natively support the + // binary type. So, we can't fully roundtrip. However, the other direction + // works: binary will be rendered in JSON, as a base64 string. So, we make + // calls to the encoder directly here, to construct a message, and one of + // these calls is ::HandleBinary, to which we pass a "binary" string + // containing "Hello, world.". + std::vector encoded; + Status status; + std::unique_ptr encoder = + NewCBOREncoder(&encoded, &status); + encoder->HandleMapBegin(); + // Emit a key. + std::vector key = {'f', 'o', 'o'}; + encoder->HandleString16(SpanFrom(key)); + // Emit the binary payload, an arbitrary array of bytes that happens to + // be the ascii message "Hello, world.". + encoder->HandleBinary(SpanFrom(std::vector{ + 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '.'})); + encoder->HandleMapEnd(); + EXPECT_EQ(Error::OK, status.error); + + // Now drive the json writer via the CBOR decoder. + std::string decoded; + std::unique_ptr json_writer = + NewJSONEncoder(&GetTestPlatform(), &decoded, &status); + ParseCBOR(SpanFrom(encoded), json_writer.get()); + EXPECT_EQ(Error::OK, status.error); + EXPECT_EQ(Status::npos(), status.pos); + // "Hello, world." in base64 is "SGVsbG8sIHdvcmxkLg==". + EXPECT_EQ("{\"foo\":\"SGVsbG8sIHdvcmxkLg==\"}", decoded); +} + +// ============================================================================= +// cbor::ParseCBOR - for receiving streaming parser events for CBOR messages +// ============================================================================= + +TEST(ParseCBORTest, ParseEmptyCBORMessage) { + // An envelope starting with 0xd8, 0x5a, with the byte length + // of 2, containing a map that's empty (0xbf for map + // start, and 0xff for map end). + std::vector in = {0xd8, 0x5a, 0, 0, 0, 2, 0xbf, 0xff}; + std::string out; + Status status; + std::unique_ptr json_writer = + NewJSONEncoder(&GetTestPlatform(), &out, &status); + ParseCBOR(span(in.data(), in.size()), json_writer.get()); + EXPECT_EQ(Error::OK, status.error); + EXPECT_EQ("{}", out); +} + +TEST(ParseCBORTest, ParseCBORHelloWorld) { + const uint8_t kPayloadLen = 27; + std::vector bytes = {0xd8, 0x5a, 0, 0, 0, kPayloadLen}; + bytes.push_back(0xbf); // start indef length map. + EncodeString8(SpanFrom("msg"), &bytes); // key: msg + // Now write the value, the familiar "Hello, 🌎." where the globe is expressed + // as two utf16 chars. + bytes.push_back(/*major type=*/2 << 5 | /*additional info=*/20); + for (uint8_t ch : std::array{ + {'H', 0, 'e', 0, 'l', 0, 'l', 0, 'o', 0, + ',', 0, ' ', 0, 0x3c, 0xd8, 0x0e, 0xdf, '.', 0}}) + bytes.push_back(ch); + bytes.push_back(0xff); // stop byte + EXPECT_EQ(kPayloadLen, bytes.size() - 6); + + std::string out; + Status status; + std::unique_ptr json_writer = + NewJSONEncoder(&GetTestPlatform(), &out, &status); + ParseCBOR(span(bytes.data(), bytes.size()), json_writer.get()); + EXPECT_EQ(Error::OK, status.error); + EXPECT_EQ("{\"msg\":\"Hello, \\ud83c\\udf0e.\"}", out); +} + +TEST(ParseCBORTest, UTF8IsSupportedInKeys) { + const uint8_t kPayloadLen = 11; + std::vector bytes = {cbor::InitialByteForEnvelope(), + cbor::InitialByteFor32BitLengthByteString(), + 0, + 0, + 0, + kPayloadLen}; + bytes.push_back(cbor::EncodeIndefiniteLengthMapStart()); + // Two UTF16 chars. + EncodeString8(SpanFrom("🌎"), &bytes); + // Can be encoded as a single UTF16 char. + EncodeString8(SpanFrom("☾"), &bytes); + bytes.push_back(cbor::EncodeStop()); + EXPECT_EQ(kPayloadLen, bytes.size() - 6); + + std::string out; + Status status; + std::unique_ptr json_writer = + NewJSONEncoder(&GetTestPlatform(), &out, &status); + ParseCBOR(span(bytes.data(), bytes.size()), json_writer.get()); + EXPECT_EQ(Error::OK, status.error); + EXPECT_EQ("{\"\\ud83c\\udf0e\":\"\\u263e\"}", out); +} + +TEST(ParseCBORTest, NoInputError) { + std::vector in = {}; + std::string out; + Status status; + std::unique_ptr json_writer = + NewJSONEncoder(&GetTestPlatform(), &out, &status); + ParseCBOR(span(in.data(), in.size()), json_writer.get()); + EXPECT_EQ(Error::CBOR_NO_INPUT, status.error); + EXPECT_EQ("", out); +} + +TEST(ParseCBORTest, InvalidStartByteError) { + // Here we test that some actual json, which usually starts with {, + // is not considered CBOR. CBOR messages must start with 0x5a, the + // envelope start byte. + std::string json = "{\"msg\": \"Hello, world.\"}"; + std::string out; + Status status; + std::unique_ptr json_writer = + NewJSONEncoder(&GetTestPlatform(), &out, &status); + ParseCBOR(SpanFrom(json), json_writer.get()); + EXPECT_EQ(Error::CBOR_INVALID_START_BYTE, status.error); + EXPECT_EQ("", out); +} + +TEST(ParseCBORTest, UnexpectedEofExpectedValueError) { + constexpr uint8_t kPayloadLen = 5; + std::vector bytes = {0xd8, 0x5a, 0, 0, 0, kPayloadLen, // envelope + 0xbf}; // map start + // A key; so value would be next. + EncodeString8(SpanFrom("key"), &bytes); + EXPECT_EQ(kPayloadLen, bytes.size() - 6); + std::string out; + Status status; + std::unique_ptr json_writer = + NewJSONEncoder(&GetTestPlatform(), &out, &status); + ParseCBOR(span(bytes.data(), bytes.size()), json_writer.get()); + EXPECT_EQ(Error::CBOR_UNEXPECTED_EOF_EXPECTED_VALUE, status.error); + EXPECT_EQ(bytes.size(), status.pos); + EXPECT_EQ("", out); +} + +TEST(ParseCBORTest, UnexpectedEofInArrayError) { + constexpr uint8_t kPayloadLen = 8; + std::vector bytes = {0xd8, 0x5a, 0, 0, 0, kPayloadLen, // envelope + 0xbf}; // The byte for starting a map. + // A key; so value would be next. + EncodeString8(SpanFrom("array"), &bytes); + bytes.push_back(0x9f); // byte for indefinite length array start. + EXPECT_EQ(kPayloadLen, bytes.size() - 6); + std::string out; + Status status; + std::unique_ptr json_writer = + NewJSONEncoder(&GetTestPlatform(), &out, &status); + ParseCBOR(span(bytes.data(), bytes.size()), json_writer.get()); + EXPECT_EQ(Error::CBOR_UNEXPECTED_EOF_IN_ARRAY, status.error); + EXPECT_EQ(bytes.size(), status.pos); + EXPECT_EQ("", out); +} + +TEST(ParseCBORTest, UnexpectedEofInMapError) { + constexpr uint8_t kPayloadLen = 1; + std::vector bytes = {0xd8, 0x5a, 0, 0, 0, kPayloadLen, // envelope + 0xbf}; // The byte for starting a map. + EXPECT_EQ(kPayloadLen, bytes.size() - 6); + std::string out; + Status status; + std::unique_ptr json_writer = + NewJSONEncoder(&GetTestPlatform(), &out, &status); + ParseCBOR(span(bytes.data(), bytes.size()), json_writer.get()); + EXPECT_EQ(Error::CBOR_UNEXPECTED_EOF_IN_MAP, status.error); + EXPECT_EQ(7u, status.pos); + EXPECT_EQ("", out); +} + +TEST(ParseCBORTest, InvalidMapKeyError) { + constexpr uint8_t kPayloadLen = 2; + std::vector bytes = {0xd8, 0x5a, 0, + 0, 0, kPayloadLen, // envelope + 0xbf, // map start + 7 << 5 | 22}; // null (not a valid map key) + EXPECT_EQ(kPayloadLen, bytes.size() - 6); + std::string out; + Status status; + std::unique_ptr json_writer = + NewJSONEncoder(&GetTestPlatform(), &out, &status); + ParseCBOR(span(bytes.data(), bytes.size()), json_writer.get()); + EXPECT_EQ(Error::CBOR_INVALID_MAP_KEY, status.error); + EXPECT_EQ(7u, status.pos); + EXPECT_EQ("", out); +} + +std::vector MakeNestedCBOR(int depth) { + std::vector bytes; + std::vector envelopes; + for (int ii = 0; ii < depth; ++ii) { + envelopes.emplace_back(); + envelopes.back().EncodeStart(&bytes); + bytes.push_back(0xbf); // indef length map start + EncodeString8(SpanFrom("key"), &bytes); + } + EncodeString8(SpanFrom("innermost_value"), &bytes); + for (int ii = 0; ii < depth; ++ii) { + bytes.push_back(0xff); // stop byte, finishes map. + envelopes.back().EncodeStop(&bytes); + envelopes.pop_back(); + } + return bytes; +} + +TEST(ParseCBORTest, StackLimitExceededError) { + { // Depth 3: no stack limit exceeded error and is easy to inspect. + std::vector bytes = MakeNestedCBOR(3); + std::string out; + Status status; + std::unique_ptr json_writer = + NewJSONEncoder(&GetTestPlatform(), &out, &status); + ParseCBOR(span(bytes.data(), bytes.size()), json_writer.get()); + EXPECT_EQ(Error::OK, status.error); + EXPECT_EQ(Status::npos(), status.pos); + EXPECT_EQ("{\"key\":{\"key\":{\"key\":\"innermost_value\"}}}", out); + } + { // Depth 300: no stack limit exceeded. + std::vector bytes = MakeNestedCBOR(300); + std::string out; + Status status; + std::unique_ptr json_writer = + NewJSONEncoder(&GetTestPlatform(), &out, &status); + ParseCBOR(span(bytes.data(), bytes.size()), json_writer.get()); + EXPECT_EQ(Error::OK, status.error); + EXPECT_EQ(Status::npos(), status.pos); + } + + // We just want to know the length of one opening map so we can compute + // where the error is encountered. So we look at a small example and find + // the second envelope start. + std::vector small_example = MakeNestedCBOR(3); + size_t opening_segment_size = 1; // Start after the first envelope start. + while (opening_segment_size < small_example.size() && + small_example[opening_segment_size] != 0xd8) + opening_segment_size++; + + { // Depth 301: limit exceeded. + std::vector bytes = MakeNestedCBOR(301); + std::string out; + Status status; + std::unique_ptr json_writer = + NewJSONEncoder(&GetTestPlatform(), &out, &status); + ParseCBOR(span(bytes.data(), bytes.size()), json_writer.get()); + EXPECT_EQ(Error::CBOR_STACK_LIMIT_EXCEEDED, status.error); + EXPECT_EQ(opening_segment_size * 301, status.pos); + } + { // Depth 320: still limit exceeded, and at the same pos as for 1001 + std::vector bytes = MakeNestedCBOR(320); + std::string out; + Status status; + std::unique_ptr json_writer = + NewJSONEncoder(&GetTestPlatform(), &out, &status); + ParseCBOR(span(bytes.data(), bytes.size()), json_writer.get()); + EXPECT_EQ(Error::CBOR_STACK_LIMIT_EXCEEDED, status.error); + EXPECT_EQ(opening_segment_size * 301, status.pos); + } +} + +TEST(ParseCBORTest, UnsupportedValueError) { + constexpr uint8_t kPayloadLen = 6; + std::vector bytes = {0xd8, 0x5a, 0, 0, 0, kPayloadLen, // envelope + 0xbf}; // map start + EncodeString8(SpanFrom("key"), &bytes); + size_t error_pos = bytes.size(); + bytes.push_back(6 << 5 | 5); // tags aren't supported yet. + EXPECT_EQ(kPayloadLen, bytes.size() - 6); + + std::string out; + Status status; + std::unique_ptr json_writer = + NewJSONEncoder(&GetTestPlatform(), &out, &status); + ParseCBOR(span(bytes.data(), bytes.size()), json_writer.get()); + EXPECT_EQ(Error::CBOR_UNSUPPORTED_VALUE, status.error); + EXPECT_EQ(error_pos, status.pos); + EXPECT_EQ("", out); +} + +TEST(ParseCBORTest, InvalidString16Error) { + constexpr uint8_t kPayloadLen = 11; + std::vector bytes = {0xd8, 0x5a, 0, 0, 0, kPayloadLen, // envelope + 0xbf}; // map start + EncodeString8(SpanFrom("key"), &bytes); + size_t error_pos = bytes.size(); + // a BYTE_STRING of length 5 as value; since we interpret these as string16, + // it's going to be invalid as each character would need two bytes, but + // 5 isn't divisible by 2. + bytes.push_back(2 << 5 | 5); + for (int ii = 0; ii < 5; ++ii) + bytes.push_back(' '); + EXPECT_EQ(kPayloadLen, bytes.size() - 6); + std::string out; + Status status; + std::unique_ptr json_writer = + NewJSONEncoder(&GetTestPlatform(), &out, &status); + ParseCBOR(span(bytes.data(), bytes.size()), json_writer.get()); + EXPECT_EQ(Error::CBOR_INVALID_STRING16, status.error); + EXPECT_EQ(error_pos, status.pos); + EXPECT_EQ("", out); +} + +TEST(ParseCBORTest, InvalidString8Error) { + constexpr uint8_t kPayloadLen = 6; + std::vector bytes = {0xd8, 0x5a, 0, 0, 0, kPayloadLen, // envelope + 0xbf}; // map start + EncodeString8(SpanFrom("key"), &bytes); + size_t error_pos = bytes.size(); + // a STRING of length 5 as value, but we're at the end of the bytes array + // so it can't be decoded successfully. + bytes.push_back(3 << 5 | 5); + EXPECT_EQ(kPayloadLen, bytes.size() - 6); + std::string out; + Status status; + std::unique_ptr json_writer = + NewJSONEncoder(&GetTestPlatform(), &out, &status); + ParseCBOR(span(bytes.data(), bytes.size()), json_writer.get()); + EXPECT_EQ(Error::CBOR_INVALID_STRING8, status.error); + EXPECT_EQ(error_pos, status.pos); + EXPECT_EQ("", out); +} + +TEST(ParseCBORTest, InvalidBinaryError) { + constexpr uint8_t kPayloadLen = 9; + std::vector bytes = {0xd8, 0x5a, 0, 0, 0, kPayloadLen, // envelope + 0xbf}; // map start + EncodeString8(SpanFrom("key"), &bytes); + size_t error_pos = bytes.size(); + bytes.push_back(6 << 5 | 22); // base64 hint for JSON; indicates binary + bytes.push_back(2 << 5 | 10); // BYTE_STRING (major type 2) of length 10 + // Just two garbage bytes, not enough for the binary. + bytes.push_back(0x31); + bytes.push_back(0x23); + EXPECT_EQ(kPayloadLen, bytes.size() - 6); + std::string out; + Status status; + std::unique_ptr json_writer = + NewJSONEncoder(&GetTestPlatform(), &out, &status); + ParseCBOR(span(bytes.data(), bytes.size()), json_writer.get()); + EXPECT_EQ(Error::CBOR_INVALID_BINARY, status.error); + EXPECT_EQ(error_pos, status.pos); + EXPECT_EQ("", out); +} + +TEST(ParseCBORTest, InvalidDoubleError) { + constexpr uint8_t kPayloadLen = 8; + std::vector bytes = {0xd8, 0x5a, 0, 0, 0, kPayloadLen, // envelope + 0xbf}; // map start + EncodeString8(SpanFrom("key"), &bytes); + size_t error_pos = bytes.size(); + bytes.push_back(7 << 5 | 27); // initial byte for double + // Just two garbage bytes, not enough to represent an actual double. + bytes.push_back(0x31); + bytes.push_back(0x23); + EXPECT_EQ(kPayloadLen, bytes.size() - 6); + std::string out; + Status status; + std::unique_ptr json_writer = + NewJSONEncoder(&GetTestPlatform(), &out, &status); + ParseCBOR(span(bytes.data(), bytes.size()), json_writer.get()); + EXPECT_EQ(Error::CBOR_INVALID_DOUBLE, status.error); + EXPECT_EQ(error_pos, status.pos); + EXPECT_EQ("", out); +} + +TEST(ParseCBORTest, InvalidSignedError) { + constexpr uint8_t kPayloadLen = 14; + std::vector bytes = {0xd8, 0x5a, 0, 0, 0, kPayloadLen, // envelope + 0xbf}; // map start + EncodeString8(SpanFrom("key"), &bytes); + size_t error_pos = bytes.size(); + // uint64_t max is a perfectly fine value to encode as CBOR unsigned, + // but we don't support this since we only cover the int32_t range. + internals::WriteTokenStart(MajorType::UNSIGNED, + std::numeric_limits::max(), &bytes); + EXPECT_EQ(kPayloadLen, bytes.size() - 6); + std::string out; + Status status; + std::unique_ptr json_writer = + NewJSONEncoder(&GetTestPlatform(), &out, &status); + ParseCBOR(span(bytes.data(), bytes.size()), json_writer.get()); + EXPECT_EQ(Error::CBOR_INVALID_INT32, status.error); + EXPECT_EQ(error_pos, status.pos); + EXPECT_EQ("", out); +} + +TEST(ParseCBORTest, TrailingJunk) { + constexpr uint8_t kPayloadLen = 35; + std::vector bytes = {0xd8, 0x5a, 0, 0, 0, kPayloadLen, // envelope + 0xbf}; // map start + EncodeString8(SpanFrom("key"), &bytes); + EncodeString8(SpanFrom("value"), &bytes); + bytes.push_back(0xff); // Up to here, it's a perfectly fine msg. + size_t error_pos = bytes.size(); + EncodeString8(SpanFrom("trailing junk"), &bytes); + + internals::WriteTokenStart(MajorType::UNSIGNED, + std::numeric_limits::max(), &bytes); + EXPECT_EQ(kPayloadLen, bytes.size() - 6); + std::string out; + Status status; + std::unique_ptr json_writer = + NewJSONEncoder(&GetTestPlatform(), &out, &status); + ParseCBOR(span(bytes.data(), bytes.size()), json_writer.get()); + EXPECT_EQ(Error::CBOR_TRAILING_JUNK, status.error); + EXPECT_EQ(error_pos, status.pos); + EXPECT_EQ("", out); +} + +// ============================================================================= +// cbor::AppendString8EntryToMap - for limited in-place editing of messages +// ============================================================================= + +template +class AppendString8EntryToMapTest : public ::testing::Test {}; + +using ContainerTestTypes = ::testing::Types, std::string>; +TYPED_TEST_SUITE(AppendString8EntryToMapTest, ContainerTestTypes); + +TYPED_TEST(AppendString8EntryToMapTest, AppendsEntrySuccessfully) { + constexpr uint8_t kPayloadLen = 12; + std::vector bytes = {0xd8, 0x5a, 0, 0, 0, kPayloadLen, // envelope + 0xbf}; // map start + size_t pos_before_payload = bytes.size() - 1; + EncodeString8(SpanFrom("key"), &bytes); + EncodeString8(SpanFrom("value"), &bytes); + bytes.push_back(0xff); // A perfectly fine cbor message. + EXPECT_EQ(kPayloadLen, bytes.size() - pos_before_payload); + + TypeParam msg(bytes.begin(), bytes.end()); + + Status status = + AppendString8EntryToCBORMap(SpanFrom("foo"), SpanFrom("bar"), &msg); + EXPECT_EQ(Error::OK, status.error); + EXPECT_EQ(Status::npos(), status.pos); + std::string out; + std::unique_ptr json_writer = + NewJSONEncoder(&GetTestPlatform(), &out, &status); + ParseCBOR(SpanFrom(msg), json_writer.get()); + EXPECT_EQ("{\"key\":\"value\",\"foo\":\"bar\"}", out); + EXPECT_EQ(Error::OK, status.error); + EXPECT_EQ(Status::npos(), status.pos); +} + +TYPED_TEST(AppendString8EntryToMapTest, AppendThreeEntries) { + std::vector encoded = { + 0xd8, 0x5a, 0, 0, 0, 2, EncodeIndefiniteLengthMapStart(), EncodeStop()}; + EXPECT_EQ(Error::OK, AppendString8EntryToCBORMap(SpanFrom("key"), + SpanFrom("value"), &encoded) + .error); + EXPECT_EQ(Error::OK, AppendString8EntryToCBORMap(SpanFrom("key1"), + SpanFrom("value1"), &encoded) + .error); + EXPECT_EQ(Error::OK, AppendString8EntryToCBORMap(SpanFrom("key2"), + SpanFrom("value2"), &encoded) + .error); + TypeParam msg(encoded.begin(), encoded.end()); + std::string out; + Status status; + std::unique_ptr json_writer = + NewJSONEncoder(&GetTestPlatform(), &out, &status); + ParseCBOR(SpanFrom(msg), json_writer.get()); + EXPECT_EQ("{\"key\":\"value\",\"key1\":\"value1\",\"key2\":\"value2\"}", out); + EXPECT_EQ(Error::OK, status.error); + EXPECT_EQ(Status::npos(), status.pos); +} + +TYPED_TEST(AppendString8EntryToMapTest, MapStartExpected_Error) { + std::vector bytes = { + 0xd8, 0x5a, 0, 0, 0, 1, EncodeIndefiniteLengthArrayStart()}; + TypeParam msg(bytes.begin(), bytes.end()); + Status status = + AppendString8EntryToCBORMap(SpanFrom("key"), SpanFrom("value"), &msg); + EXPECT_EQ(Error::CBOR_MAP_START_EXPECTED, status.error); + EXPECT_EQ(6u, status.pos); +} + +TYPED_TEST(AppendString8EntryToMapTest, MapStopExpected_Error) { + std::vector bytes = { + 0xd8, 0x5a, 0, 0, 0, 2, EncodeIndefiniteLengthMapStart(), 42}; + TypeParam msg(bytes.begin(), bytes.end()); + Status status = + AppendString8EntryToCBORMap(SpanFrom("key"), SpanFrom("value"), &msg); + EXPECT_EQ(Error::CBOR_MAP_STOP_EXPECTED, status.error); + EXPECT_EQ(7u, status.pos); +} + +TYPED_TEST(AppendString8EntryToMapTest, InvalidEnvelope_Error) { + { // Second byte is wrong. + std::vector bytes = { + 0x5a, 0, 0, 0, 2, EncodeIndefiniteLengthMapStart(), EncodeStop(), 0}; + TypeParam msg(bytes.begin(), bytes.end()); + Status status = + AppendString8EntryToCBORMap(SpanFrom("key"), SpanFrom("value"), &msg); + EXPECT_EQ(Error::CBOR_INVALID_ENVELOPE, status.error); + EXPECT_EQ(0u, status.pos); + } + { // Second byte is wrong. + std::vector bytes = { + 0xd8, 0x7a, 0, 0, 0, 2, EncodeIndefiniteLengthMapStart(), EncodeStop()}; + TypeParam msg(bytes.begin(), bytes.end()); + Status status = + AppendString8EntryToCBORMap(SpanFrom("key"), SpanFrom("value"), &msg); + EXPECT_EQ(Error::CBOR_INVALID_ENVELOPE, status.error); + EXPECT_EQ(0u, status.pos); + } + { // Invalid envelope size example. + std::vector bytes = { + 0xd8, 0x5a, 0, 0, 0, 3, EncodeIndefiniteLengthMapStart(), EncodeStop(), + }; + TypeParam msg(bytes.begin(), bytes.end()); + Status status = + AppendString8EntryToCBORMap(SpanFrom("key"), SpanFrom("value"), &msg); + EXPECT_EQ(Error::CBOR_INVALID_ENVELOPE, status.error); + EXPECT_EQ(0u, status.pos); + } + { // Invalid envelope size example. + std::vector bytes = { + 0xd8, 0x5a, 0, 0, 0, 1, EncodeIndefiniteLengthMapStart(), EncodeStop(), + }; + TypeParam msg(bytes.begin(), bytes.end()); + Status status = + AppendString8EntryToCBORMap(SpanFrom("key"), SpanFrom("value"), &msg); + EXPECT_EQ(Error::CBOR_INVALID_ENVELOPE, status.error); + EXPECT_EQ(0u, status.pos); + } +} +} // namespace cbor + +namespace json { + +// ============================================================================= +// json::NewJSONEncoder - for encoding streaming parser events as JSON +// ============================================================================= + +void WriteUTF8AsUTF16(StreamingParserHandler* writer, const std::string& utf8) { + writer->HandleString16(SpanFrom(UTF8ToUTF16(SpanFrom(utf8)))); +} + +TEST(JsonStdStringWriterTest, HelloWorld) { + std::string out; + Status status; + std::unique_ptr writer = + NewJSONEncoder(&GetTestPlatform(), &out, &status); + writer->HandleMapBegin(); + WriteUTF8AsUTF16(writer.get(), "msg1"); + WriteUTF8AsUTF16(writer.get(), "Hello, 🌎."); + std::string key = "msg1-as-utf8"; + std::string value = "Hello, 🌎."; + writer->HandleString8(SpanFrom(key)); + writer->HandleString8(SpanFrom(value)); + WriteUTF8AsUTF16(writer.get(), "msg2"); + WriteUTF8AsUTF16(writer.get(), "\\\b\r\n\t\f\""); + WriteUTF8AsUTF16(writer.get(), "nested"); + writer->HandleMapBegin(); + WriteUTF8AsUTF16(writer.get(), "double"); + writer->HandleDouble(3.1415); + WriteUTF8AsUTF16(writer.get(), "int"); + writer->HandleInt32(-42); + WriteUTF8AsUTF16(writer.get(), "bool"); + writer->HandleBool(false); + WriteUTF8AsUTF16(writer.get(), "null"); + writer->HandleNull(); + writer->HandleMapEnd(); + WriteUTF8AsUTF16(writer.get(), "array"); + writer->HandleArrayBegin(); + writer->HandleInt32(1); + writer->HandleInt32(2); + writer->HandleInt32(3); + writer->HandleArrayEnd(); + writer->HandleMapEnd(); + EXPECT_TRUE(status.ok()); + EXPECT_EQ( + "{\"msg1\":\"Hello, \\ud83c\\udf0e.\"," + "\"msg1-as-utf8\":\"Hello, \\ud83c\\udf0e.\"," + "\"msg2\":\"\\\\\\b\\r\\n\\t\\f\\\"\"," + "\"nested\":{\"double\":3.1415,\"int\":-42," + "\"bool\":false,\"null\":null},\"array\":[1,2,3]}", + out); +} + +TEST(JsonStdStringWriterTest, RepresentingNonFiniteValuesAsNull) { + // JSON can't represent +Infinity, -Infinity, or NaN. + // So in practice it's mapped to null. + std::string out; + Status status; + std::unique_ptr writer = + NewJSONEncoder(&GetTestPlatform(), &out, &status); + writer->HandleMapBegin(); + writer->HandleString8(SpanFrom("Infinity")); + writer->HandleDouble(std::numeric_limits::infinity()); + writer->HandleString8(SpanFrom("-Infinity")); + writer->HandleDouble(-std::numeric_limits::infinity()); + writer->HandleString8(SpanFrom("NaN")); + writer->HandleDouble(std::numeric_limits::quiet_NaN()); + writer->HandleMapEnd(); + EXPECT_TRUE(status.ok()); + EXPECT_EQ("{\"Infinity\":null,\"-Infinity\":null,\"NaN\":null}", out); +} + +TEST(JsonStdStringWriterTest, BinaryEncodedAsJsonString) { + // The encoder emits binary submitted to StreamingParserHandler::HandleBinary + // as base64. The following three examples are taken from + // https://en.wikipedia.org/wiki/Base64. + { + std::string out; + Status status; + std::unique_ptr writer = + NewJSONEncoder(&GetTestPlatform(), &out, &status); + writer->HandleBinary(SpanFrom(std::vector({'M', 'a', 'n'}))); + EXPECT_TRUE(status.ok()); + EXPECT_EQ("\"TWFu\"", out); + } + { + std::string out; + Status status; + std::unique_ptr writer = + NewJSONEncoder(&GetTestPlatform(), &out, &status); + writer->HandleBinary(SpanFrom(std::vector({'M', 'a'}))); + EXPECT_TRUE(status.ok()); + EXPECT_EQ("\"TWE=\"", out); + } + { + std::string out; + Status status; + std::unique_ptr writer = + NewJSONEncoder(&GetTestPlatform(), &out, &status); + writer->HandleBinary(SpanFrom(std::vector({'M'}))); + EXPECT_TRUE(status.ok()); + EXPECT_EQ("\"TQ==\"", out); + } + { // "Hello, world.", verified with base64decode.org. + std::string out; + Status status; + std::unique_ptr writer = + NewJSONEncoder(&GetTestPlatform(), &out, &status); + writer->HandleBinary(SpanFrom(std::vector( + {'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '.'}))); + EXPECT_TRUE(status.ok()); + EXPECT_EQ("\"SGVsbG8sIHdvcmxkLg==\"", out); + } +} + +TEST(JsonStdStringWriterTest, HandlesErrors) { + // When an error is sent via HandleError, it saves it in the provided + // status and clears the output. + std::string out; + Status status; + std::unique_ptr writer = + NewJSONEncoder(&GetTestPlatform(), &out, &status); + writer->HandleMapBegin(); + WriteUTF8AsUTF16(writer.get(), "msg1"); + writer->HandleError(Status{Error::JSON_PARSER_VALUE_EXPECTED, 42}); + EXPECT_EQ(Error::JSON_PARSER_VALUE_EXPECTED, status.error); + EXPECT_EQ(42u, status.pos); + EXPECT_EQ("", out); +} + +// We'd use Gmock but unfortunately it only handles copyable return types. +class MockPlatform : public Platform { + public: + // Not implemented. + bool StrToD(const char* str, double* result) const override { return false; } + + // A map with pre-registered responses for DToSTr. + std::map dtostr_responses_; + + std::unique_ptr DToStr(double value) const override { + auto it = dtostr_responses_.find(value); + CHECK(it != dtostr_responses_.end()); + const std::string& str = it->second; + std::unique_ptr response(new char[str.size() + 1]); + memcpy(response.get(), str.c_str(), str.size() + 1); + return response; + } +}; + +TEST(JsonStdStringWriterTest, DoubleToString) { + // This "broken" platform responds without the leading 0 before the + // decimal dot, so it'd be invalid JSON. + MockPlatform platform; + platform.dtostr_responses_[.1] = ".1"; + platform.dtostr_responses_[-.7] = "-.7"; + + std::string out; + Status status; + std::unique_ptr writer = + NewJSONEncoder(&platform, &out, &status); + writer->HandleArrayBegin(); + writer->HandleDouble(.1); + writer->HandleDouble(-.7); + writer->HandleArrayEnd(); + EXPECT_EQ("[0.1,-0.7]", out); +} + +// ============================================================================= +// json::ParseJSON - for receiving streaming parser events for JSON +// ============================================================================= + +class Log : public StreamingParserHandler { + public: + void HandleMapBegin() override { log_ << "map begin\n"; } + + void HandleMapEnd() override { log_ << "map end\n"; } + + void HandleArrayBegin() override { log_ << "array begin\n"; } + + void HandleArrayEnd() override { log_ << "array end\n"; } + + void HandleString8(span chars) override { + log_ << "string8: " << std::string(chars.begin(), chars.end()) << "\n"; + } + + void HandleString16(span chars) override { + log_ << "string16: " << UTF16ToUTF8(chars) << "\n"; + } + + void HandleBinary(span bytes) override { + // JSON doesn't have native support for arbitrary bytes, so our parser will + // never call this. + CHECK(false); + } + + void HandleDouble(double value) override { + log_ << "double: " << value << "\n"; + } + + void HandleInt32(int32_t value) override { log_ << "int: " << value << "\n"; } + + void HandleBool(bool value) override { log_ << "bool: " << value << "\n"; } + + void HandleNull() override { log_ << "null\n"; } + + void HandleError(Status status) override { status_ = status; } + + std::string str() const { return status_.ok() ? log_.str() : ""; } + + Status status() const { return status_; } + + private: + std::ostringstream log_; + Status status_; +}; + +class JsonParserTest : public ::testing::Test { + protected: + Log log_; +}; + +TEST_F(JsonParserTest, SimpleDictionary) { + std::string json = "{\"foo\": 42}"; + ParseJSON(GetTestPlatform(), SpanFrom(json), &log_); + EXPECT_TRUE(log_.status().ok()); + EXPECT_EQ( + "map begin\n" + "string16: foo\n" + "int: 42\n" + "map end\n", + log_.str()); +} + +TEST_F(JsonParserTest, Whitespace) { + std::string json = "\n {\n\"msg\"\n: \v\"Hello, world.\"\t\r}\t"; + ParseJSON(GetTestPlatform(), SpanFrom(json), &log_); + EXPECT_TRUE(log_.status().ok()); + EXPECT_EQ( + "map begin\n" + "string16: msg\n" + "string16: Hello, world.\n" + "map end\n", + log_.str()); +} + +TEST_F(JsonParserTest, NestedDictionary) { + std::string json = "{\"foo\": {\"bar\": {\"baz\": 1}, \"bar2\": 2}}"; + ParseJSON(GetTestPlatform(), SpanFrom(json), &log_); + EXPECT_TRUE(log_.status().ok()); + EXPECT_EQ( + "map begin\n" + "string16: foo\n" + "map begin\n" + "string16: bar\n" + "map begin\n" + "string16: baz\n" + "int: 1\n" + "map end\n" + "string16: bar2\n" + "int: 2\n" + "map end\n" + "map end\n", + log_.str()); +} + +TEST_F(JsonParserTest, Doubles) { + std::string json = "{\"foo\": 3.1415, \"bar\": 31415e-4}"; + ParseJSON(GetTestPlatform(), SpanFrom(json), &log_); + EXPECT_TRUE(log_.status().ok()); + EXPECT_EQ( + "map begin\n" + "string16: foo\n" + "double: 3.1415\n" + "string16: bar\n" + "double: 3.1415\n" + "map end\n", + log_.str()); +} + +TEST_F(JsonParserTest, Unicode) { + // Globe character. 0xF0 0x9F 0x8C 0x8E in utf8, 0xD83C 0xDF0E in utf16. + std::string json = "{\"msg\": \"Hello, \\uD83C\\uDF0E.\"}"; + ParseJSON(GetTestPlatform(), SpanFrom(json), &log_); + EXPECT_TRUE(log_.status().ok()); + EXPECT_EQ( + "map begin\n" + "string16: msg\n" + "string16: Hello, 🌎.\n" + "map end\n", + log_.str()); +} + +TEST_F(JsonParserTest, Unicode_ParseUtf16) { + // Globe character. utf8: 0xF0 0x9F 0x8C 0x8E; utf16: 0xD83C 0xDF0E. + // Crescent moon character. utf8: 0xF0 0x9F 0x8C 0x99; utf16: 0xD83C 0xDF19. + + // We provide the moon with json escape, but the earth as utf16 input. + // Either way they arrive as utf8 (after decoding in log_.str()). + std::vector json = + UTF8ToUTF16(SpanFrom("{\"space\": \"🌎 \\uD83C\\uDF19.\"}")); + ParseJSON(GetTestPlatform(), SpanFrom(json), &log_); + EXPECT_TRUE(log_.status().ok()); + EXPECT_EQ( + "map begin\n" + "string16: space\n" + "string16: 🌎 🌙.\n" + "map end\n", + log_.str()); +} + +TEST_F(JsonParserTest, Unicode_ParseUtf8) { + // Used below: + // гласность - example for 2 byte utf8, Russian word "glasnost" + // 屋 - example for 3 byte utf8, Chinese word for "house" + // 🌎 - example for 4 byte utf8: 0xF0 0x9F 0x8C 0x8E; utf16: 0xD83C 0xDF0E. + // 🌙 - example for escapes: utf8: 0xF0 0x9F 0x8C 0x99; utf16: 0xD83C 0xDF19. + + // We provide the moon with json escape, but the earth as utf8 input. + // Either way they arrive as utf8 (after decoding in log_.str()). + std::string json = + "{" + "\"escapes\": \"\\uD83C\\uDF19\"," + "\"2 byte\":\"гласность\"," + "\"3 byte\":\"屋\"," + "\"4 byte\":\"🌎\"" + "}"; + ParseJSON(GetTestPlatform(), SpanFrom(json), &log_); + EXPECT_TRUE(log_.status().ok()); + EXPECT_EQ( + "map begin\n" + "string16: escapes\n" + "string16: 🌙\n" + "string16: 2 byte\n" + "string16: гласность\n" + "string16: 3 byte\n" + "string16: 屋\n" + "string16: 4 byte\n" + "string16: 🌎\n" + "map end\n", + log_.str()); +} + +TEST_F(JsonParserTest, UnprocessedInputRemainsError) { + // Trailing junk after the valid JSON. + std::string json = "{\"foo\": 3.1415} junk"; + size_t junk_idx = json.find("junk"); + EXPECT_NE(junk_idx, std::string::npos); + ParseJSON(GetTestPlatform(), SpanFrom(json), &log_); + EXPECT_EQ(Error::JSON_PARSER_UNPROCESSED_INPUT_REMAINS, log_.status().error); + EXPECT_EQ(junk_idx, log_.status().pos); + EXPECT_EQ("", log_.str()); +} + +std::string MakeNestedJson(int depth) { + std::string json; + for (int ii = 0; ii < depth; ++ii) + json += "{\"foo\":"; + json += "42"; + for (int ii = 0; ii < depth; ++ii) + json += "}"; + return json; +} + +TEST_F(JsonParserTest, StackLimitExceededError_BelowLimit) { + // kStackLimit is 300 (see json_parser.cc). First let's + // try with a small nested example. + std::string json_3 = MakeNestedJson(3); + ParseJSON(GetTestPlatform(), SpanFrom(json_3), &log_); + EXPECT_TRUE(log_.status().ok()); + EXPECT_EQ( + "map begin\n" + "string16: foo\n" + "map begin\n" + "string16: foo\n" + "map begin\n" + "string16: foo\n" + "int: 42\n" + "map end\n" + "map end\n" + "map end\n", + log_.str()); +} + +TEST_F(JsonParserTest, StackLimitExceededError_AtLimit) { + // Now with kStackLimit (300). + std::string json_limit = MakeNestedJson(300); + ParseJSON(GetTestPlatform(), + span(reinterpret_cast(json_limit.data()), + json_limit.size()), + &log_); + EXPECT_TRUE(log_.status().ok()); +} + +TEST_F(JsonParserTest, StackLimitExceededError_AboveLimit) { + // Now with kStackLimit + 1 (301) - it exceeds in the innermost instance. + std::string exceeded = MakeNestedJson(301); + ParseJSON(GetTestPlatform(), SpanFrom(exceeded), &log_); + EXPECT_EQ(Error::JSON_PARSER_STACK_LIMIT_EXCEEDED, log_.status().error); + EXPECT_EQ(strlen("{\"foo\":") * 301, log_.status().pos); +} + +TEST_F(JsonParserTest, StackLimitExceededError_WayAboveLimit) { + // Now way past the limit. Still, the point of exceeding is 301. + std::string far_out = MakeNestedJson(320); + ParseJSON(GetTestPlatform(), SpanFrom(far_out), &log_); + EXPECT_EQ(Error::JSON_PARSER_STACK_LIMIT_EXCEEDED, log_.status().error); + EXPECT_EQ(strlen("{\"foo\":") * 301, log_.status().pos); +} + +TEST_F(JsonParserTest, NoInputError) { + std::string json = ""; + ParseJSON(GetTestPlatform(), SpanFrom(json), &log_); + EXPECT_EQ(Error::JSON_PARSER_NO_INPUT, log_.status().error); + EXPECT_EQ(0u, log_.status().pos); + EXPECT_EQ("", log_.str()); +} + +TEST_F(JsonParserTest, InvalidTokenError) { + std::string json = "|"; + ParseJSON(GetTestPlatform(), SpanFrom(json), &log_); + EXPECT_EQ(Error::JSON_PARSER_INVALID_TOKEN, log_.status().error); + EXPECT_EQ(0u, log_.status().pos); + EXPECT_EQ("", log_.str()); +} + +TEST_F(JsonParserTest, InvalidNumberError) { + // Mantissa exceeds max (the constant used here is int64_t max). + std::string json = "1E9223372036854775807"; + ParseJSON(GetTestPlatform(), SpanFrom(json), &log_); + EXPECT_EQ(Error::JSON_PARSER_INVALID_NUMBER, log_.status().error); + EXPECT_EQ(0u, log_.status().pos); + EXPECT_EQ("", log_.str()); +} + +TEST_F(JsonParserTest, InvalidStringError) { + // \x22 is an unsupported escape sequence + std::string json = "\"foo\\x22\""; + ParseJSON(GetTestPlatform(), SpanFrom(json), &log_); + EXPECT_EQ(Error::JSON_PARSER_INVALID_STRING, log_.status().error); + EXPECT_EQ(0u, log_.status().pos); + EXPECT_EQ("", log_.str()); +} + +TEST_F(JsonParserTest, UnexpectedArrayEndError) { + std::string json = "[1,2,]"; + ParseJSON(GetTestPlatform(), SpanFrom(json), &log_); + EXPECT_EQ(Error::JSON_PARSER_UNEXPECTED_ARRAY_END, log_.status().error); + EXPECT_EQ(5u, log_.status().pos); + EXPECT_EQ("", log_.str()); +} + +TEST_F(JsonParserTest, CommaOrArrayEndExpectedError) { + std::string json = "[1,2 2"; + ParseJSON(GetTestPlatform(), SpanFrom(json), &log_); + EXPECT_EQ(Error::JSON_PARSER_COMMA_OR_ARRAY_END_EXPECTED, + log_.status().error); + EXPECT_EQ(5u, log_.status().pos); + EXPECT_EQ("", log_.str()); +} + +TEST_F(JsonParserTest, StringLiteralExpectedError) { + // There's an error because the key bar, a string, is not terminated. + std::string json = "{\"foo\": 3.1415, \"bar: 31415e-4}"; + ParseJSON(GetTestPlatform(), SpanFrom(json), &log_); + EXPECT_EQ(Error::JSON_PARSER_STRING_LITERAL_EXPECTED, log_.status().error); + EXPECT_EQ(16u, log_.status().pos); + EXPECT_EQ("", log_.str()); +} + +TEST_F(JsonParserTest, ColonExpectedError) { + std::string json = "{\"foo\", 42}"; + ParseJSON(GetTestPlatform(), SpanFrom(json), &log_); + EXPECT_EQ(Error::JSON_PARSER_COLON_EXPECTED, log_.status().error); + EXPECT_EQ(6u, log_.status().pos); + EXPECT_EQ("", log_.str()); +} + +TEST_F(JsonParserTest, UnexpectedMapEndError) { + std::string json = "{\"foo\": 42, }"; + ParseJSON(GetTestPlatform(), SpanFrom(json), &log_); + EXPECT_EQ(Error::JSON_PARSER_UNEXPECTED_MAP_END, log_.status().error); + EXPECT_EQ(12u, log_.status().pos); + EXPECT_EQ("", log_.str()); +} + +TEST_F(JsonParserTest, CommaOrMapEndExpectedError) { + // The second separator should be a comma. + std::string json = "{\"foo\": 3.1415: \"bar\": 0}"; + ParseJSON(GetTestPlatform(), SpanFrom(json), &log_); + EXPECT_EQ(Error::JSON_PARSER_COMMA_OR_MAP_END_EXPECTED, log_.status().error); + EXPECT_EQ(14u, log_.status().pos); + EXPECT_EQ("", log_.str()); +} + +TEST_F(JsonParserTest, ValueExpectedError) { + std::string json = "}"; + ParseJSON(GetTestPlatform(), SpanFrom(json), &log_); + EXPECT_EQ(Error::JSON_PARSER_VALUE_EXPECTED, log_.status().error); + EXPECT_EQ(0u, log_.status().pos); + EXPECT_EQ("", log_.str()); +} + +template +class ConvertJSONToCBORTest : public ::testing::Test {}; + +using ContainerTestTypes = ::testing::Types, std::string>; +TYPED_TEST_SUITE(ConvertJSONToCBORTest, ContainerTestTypes); + +TYPED_TEST(ConvertJSONToCBORTest, RoundTripValidJson) { + std::string json_in = "{\"msg\":\"Hello, world.\",\"lst\":[1,2,3]}"; + TypeParam json(json_in.begin(), json_in.end()); + TypeParam cbor; + { + Status status = ConvertJSONToCBOR(GetTestPlatform(), SpanFrom(json), &cbor); + EXPECT_EQ(Error::OK, status.error); + EXPECT_EQ(Status::npos(), status.pos); + } + TypeParam roundtrip_json; + { + Status status = + ConvertCBORToJSON(GetTestPlatform(), SpanFrom(cbor), &roundtrip_json); + EXPECT_EQ(Error::OK, status.error); + EXPECT_EQ(Status::npos(), status.pos); + } + EXPECT_EQ(json, roundtrip_json); +} + +TYPED_TEST(ConvertJSONToCBORTest, RoundTripValidJson16) { + std::vector json16 = { + '{', '"', 'm', 's', 'g', '"', ':', '"', 'H', 'e', 'l', 'l', + 'o', ',', ' ', 0xd83c, 0xdf0e, '.', '"', ',', '"', 'l', 's', 't', + '"', ':', '[', '1', ',', '2', ',', '3', ']', '}'}; + TypeParam cbor; + { + Status status = ConvertJSONToCBOR( + GetTestPlatform(), span(json16.data(), json16.size()), &cbor); + EXPECT_EQ(Error::OK, status.error); + EXPECT_EQ(Status::npos(), status.pos); + } + TypeParam roundtrip_json; + { + Status status = + ConvertCBORToJSON(GetTestPlatform(), SpanFrom(cbor), &roundtrip_json); + EXPECT_EQ(Error::OK, status.error); + EXPECT_EQ(Status::npos(), status.pos); + } + std::string json = "{\"msg\":\"Hello, \\ud83c\\udf0e.\",\"lst\":[1,2,3]}"; + TypeParam expected_json(json.begin(), json.end()); + EXPECT_EQ(expected_json, roundtrip_json); +} +} // namespace json +} // namespace v8_inspector_protocol_encoding diff --git a/tools/inspector_protocol/encoding/encoding_test_helper.h b/tools/inspector_protocol/encoding/encoding_test_helper.h new file mode 100644 index 00000000000000..84da2e72e87f5c --- /dev/null +++ b/tools/inspector_protocol/encoding/encoding_test_helper.h @@ -0,0 +1,33 @@ +// Copyright 2019 The V8 Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// This file is V8 specific, to make encoding_test.cc work. +// It is not rolled from the upstream project. + +#ifndef V8_INSPECTOR_PROTOCOL_ENCODING_ENCODING_TEST_HELPER_H_ +#define V8_INSPECTOR_PROTOCOL_ENCODING_ENCODING_TEST_HELPER_H_ + +#include +#include + +#include "src/base/logging.h" +#include "src/inspector/v8-string-conversions.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace v8_inspector_protocol_encoding { + +std::string UTF16ToUTF8(span in) { + return v8_inspector::UTF16ToUTF8(in.data(), in.size()); +} + +std::vector UTF8ToUTF16(span in) { + std::basic_string utf16 = v8_inspector::UTF8ToUTF16( + reinterpret_cast(in.data()), in.size()); + return std::vector(utf16.begin(), utf16.end()); +} + +} // namespace v8_inspector_protocol_encoding + +#endif // V8_INSPECTOR_PROTOCOL_ENCODING_ENCODING_TEST_HELPER_H_ diff --git a/tools/inspector_protocol/inspector_protocol.gni b/tools/inspector_protocol/inspector_protocol.gni index 5dcc1f522d8634..d612fb6aebb52c 100644 --- a/tools/inspector_protocol/inspector_protocol.gni +++ b/tools/inspector_protocol/inspector_protocol.gni @@ -27,13 +27,16 @@ template("inspector_protocol_generate") { inspector_protocol_dir = invoker.inspector_protocol_dir action(target_name) { - script = "$inspector_protocol_dir/CodeGenerator.py" + script = "$inspector_protocol_dir/code_generator.py" inputs = [ invoker.config_file, + "$inspector_protocol_dir/lib/base_string_adapter_cc.template", + "$inspector_protocol_dir/lib/base_string_adapter_h.template", + "$inspector_protocol_dir/lib/encoding_h.template", + "$inspector_protocol_dir/lib/encoding_cpp.template", "$inspector_protocol_dir/lib/Allocator_h.template", "$inspector_protocol_dir/lib/Array_h.template", - "$inspector_protocol_dir/lib/Collections_h.template", "$inspector_protocol_dir/lib/DispatcherBase_cpp.template", "$inspector_protocol_dir/lib/DispatcherBase_h.template", "$inspector_protocol_dir/lib/ErrorSupport_cpp.template", diff --git a/tools/inspector_protocol/inspector_protocol.gypi b/tools/inspector_protocol/inspector_protocol.gypi index 1fb7119b5fa567..d614474e69c32e 100644 --- a/tools/inspector_protocol/inspector_protocol.gypi +++ b/tools/inspector_protocol/inspector_protocol.gypi @@ -5,9 +5,10 @@ { 'variables': { 'inspector_protocol_files': [ + 'lib/encoding_h.template', + 'lib/encoding_cpp.template', 'lib/Allocator_h.template', 'lib/Array_h.template', - 'lib/Collections_h.template', 'lib/DispatcherBase_cpp.template', 'lib/DispatcherBase_h.template', 'lib/ErrorSupport_cpp.template', @@ -27,7 +28,7 @@ 'templates/Imported_h.template', 'templates/TypeBuilder_cpp.template', 'templates/TypeBuilder_h.template', - 'CodeGenerator.py', + 'code_generator.py', ] } } diff --git a/tools/inspector_protocol/lib/CBOR_cpp.template b/tools/inspector_protocol/lib/CBOR_cpp.template deleted file mode 100644 index 36750b19a3c935..00000000000000 --- a/tools/inspector_protocol/lib/CBOR_cpp.template +++ /dev/null @@ -1,803 +0,0 @@ -{# This template is generated by gen_cbor_templates.py. #} -// Generated by lib/CBOR_cpp.template. - -// Copyright 2019 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - - -#include -#include - -{% for namespace in config.protocol.namespace %} -namespace {{namespace}} { -{% endfor %} - -// ===== encoding/cbor.cc ===== - -using namespace cbor; - -namespace { - -// See RFC 7049 Section 2.3, Table 2. -static constexpr uint8_t kEncodedTrue = - EncodeInitialByte(MajorType::SIMPLE_VALUE, 21); -static constexpr uint8_t kEncodedFalse = - EncodeInitialByte(MajorType::SIMPLE_VALUE, 20); -static constexpr uint8_t kEncodedNull = - EncodeInitialByte(MajorType::SIMPLE_VALUE, 22); -static constexpr uint8_t kInitialByteForDouble = - EncodeInitialByte(MajorType::SIMPLE_VALUE, 27); - -} // namespace - -uint8_t EncodeTrue() { return kEncodedTrue; } -uint8_t EncodeFalse() { return kEncodedFalse; } -uint8_t EncodeNull() { return kEncodedNull; } - -uint8_t EncodeIndefiniteLengthArrayStart() { - return kInitialByteIndefiniteLengthArray; -} - -uint8_t EncodeIndefiniteLengthMapStart() { - return kInitialByteIndefiniteLengthMap; -} - -uint8_t EncodeStop() { return kStopByte; } - -namespace { -// See RFC 7049 Table 3 and Section 2.4.4.2. This is used as a prefix for -// arbitrary binary data encoded as BYTE_STRING. -static constexpr uint8_t kExpectedConversionToBase64Tag = - EncodeInitialByte(MajorType::TAG, 22); - -// When parsing CBOR, we limit recursion depth for objects and arrays -// to this constant. -static constexpr int kStackLimit = 1000; - -// Writes the bytes for |v| to |out|, starting with the most significant byte. -// See also: https://commandcenter.blogspot.com/2012/04/byte-order-fallacy.html -template -void WriteBytesMostSignificantByteFirst(T v, std::vector* out) { - for (int shift_bytes = sizeof(T) - 1; shift_bytes >= 0; --shift_bytes) - out->push_back(0xff & (v >> (shift_bytes * 8))); -} -} // namespace - -namespace cbor_internals { -// Writes the start of a token with |type|. The |value| may indicate the size, -// or it may be the payload if the value is an unsigned integer. -void WriteTokenStart(MajorType type, uint64_t value, - std::vector* encoded) { - if (value < 24) { - // Values 0-23 are encoded directly into the additional info of the - // initial byte. - encoded->push_back(EncodeInitialByte(type, /*additional_info=*/value)); - return; - } - if (value <= std::numeric_limits::max()) { - // Values 24-255 are encoded with one initial byte, followed by the value. - encoded->push_back(EncodeInitialByte(type, kAdditionalInformation1Byte)); - encoded->push_back(value); - return; - } - if (value <= std::numeric_limits::max()) { - // Values 256-65535: 1 initial byte + 2 bytes payload. - encoded->push_back(EncodeInitialByte(type, kAdditionalInformation2Bytes)); - WriteBytesMostSignificantByteFirst(value, encoded); - return; - } - if (value <= std::numeric_limits::max()) { - // 32 bit uint: 1 initial byte + 4 bytes payload. - encoded->push_back(EncodeInitialByte(type, kAdditionalInformation4Bytes)); - WriteBytesMostSignificantByteFirst(static_cast(value), - encoded); - return; - } - // 64 bit uint: 1 initial byte + 8 bytes payload. - encoded->push_back(EncodeInitialByte(type, kAdditionalInformation8Bytes)); - WriteBytesMostSignificantByteFirst(value, encoded); -} -} // namespace cbor_internals - -namespace { -// Extracts sizeof(T) bytes from |in| to extract a value of type T -// (e.g. uint64_t, uint32_t, ...), most significant byte first. -// See also: https://commandcenter.blogspot.com/2012/04/byte-order-fallacy.html -template -T ReadBytesMostSignificantByteFirst(span in) { - assert(static_cast(in.size()) >= sizeof(T)); - T result = 0; - for (std::size_t shift_bytes = 0; shift_bytes < sizeof(T); ++shift_bytes) - result |= T(in[sizeof(T) - 1 - shift_bytes]) << (shift_bytes * 8); - return result; -} -} // namespace - -namespace cbor_internals { -int8_t ReadTokenStart(span bytes, MajorType* type, uint64_t* value) { - if (bytes.empty()) return -1; - uint8_t initial_byte = bytes[0]; - *type = MajorType((initial_byte & kMajorTypeMask) >> kMajorTypeBitShift); - - uint8_t additional_information = initial_byte & kAdditionalInformationMask; - if (additional_information < 24) { - // Values 0-23 are encoded directly into the additional info of the - // initial byte. - *value = additional_information; - return 1; - } - if (additional_information == kAdditionalInformation1Byte) { - // Values 24-255 are encoded with one initial byte, followed by the value. - if (bytes.size() < 2) return -1; - *value = ReadBytesMostSignificantByteFirst(bytes.subspan(1)); - return 2; - } - if (additional_information == kAdditionalInformation2Bytes) { - // Values 256-65535: 1 initial byte + 2 bytes payload. - if (static_cast(bytes.size()) < 1 + sizeof(uint16_t)) - return -1; - *value = ReadBytesMostSignificantByteFirst(bytes.subspan(1)); - return 3; - } - if (additional_information == kAdditionalInformation4Bytes) { - // 32 bit uint: 1 initial byte + 4 bytes payload. - if (static_cast(bytes.size()) < 1 + sizeof(uint32_t)) - return -1; - *value = ReadBytesMostSignificantByteFirst(bytes.subspan(1)); - return 5; - } - if (additional_information == kAdditionalInformation8Bytes) { - // 64 bit uint: 1 initial byte + 8 bytes payload. - if (static_cast(bytes.size()) < 1 + sizeof(uint64_t)) - return -1; - *value = ReadBytesMostSignificantByteFirst(bytes.subspan(1)); - return 9; - } - return -1; -} -} // namespace cbor_internals - -using cbor_internals::WriteTokenStart; -using cbor_internals::ReadTokenStart; - -void EncodeInt32(int32_t value, std::vector* out) { - if (value >= 0) { - WriteTokenStart(MajorType::UNSIGNED, value, out); - } else { - uint64_t representation = static_cast(-(value + 1)); - WriteTokenStart(MajorType::NEGATIVE, representation, out); - } -} - -void EncodeString16(span in, std::vector* out) { - uint64_t byte_length = static_cast(in.size_bytes()); - WriteTokenStart(MajorType::BYTE_STRING, byte_length, out); - // When emitting UTF16 characters, we always write the least significant byte - // first; this is because it's the native representation for X86. - // TODO(johannes): Implement a more efficient thing here later, e.g. - // casting *iff* the machine has this byte order. - // The wire format for UTF16 chars will probably remain the same - // (least significant byte first) since this way we can have - // golden files, unittests, etc. that port easily and universally. - // See also: - // https://commandcenter.blogspot.com/2012/04/byte-order-fallacy.html - for (const uint16_t two_bytes : in) { - out->push_back(two_bytes); - out->push_back(two_bytes >> 8); - } -} - -void EncodeString8(span in, std::vector* out) { - WriteTokenStart(MajorType::STRING, static_cast(in.size_bytes()), - out); - out->insert(out->end(), in.begin(), in.end()); -} - -void EncodeBinary(span in, std::vector* out) { - out->push_back(kExpectedConversionToBase64Tag); - uint64_t byte_length = static_cast(in.size_bytes()); - WriteTokenStart(MajorType::BYTE_STRING, byte_length, out); - out->insert(out->end(), in.begin(), in.end()); -} - -// A double is encoded with a specific initial byte -// (kInitialByteForDouble) plus the 64 bits of payload for its value. -constexpr std::ptrdiff_t kEncodedDoubleSize = 1 + sizeof(uint64_t); - -// An envelope is encoded with a specific initial byte -// (kInitialByteForEnvelope), plus the start byte for a BYTE_STRING with a 32 -// bit wide length, plus a 32 bit length for that string. -constexpr std::ptrdiff_t kEncodedEnvelopeHeaderSize = 1 + 1 + sizeof(uint32_t); - -void EncodeDouble(double value, std::vector* out) { - // The additional_info=27 indicates 64 bits for the double follow. - // See RFC 7049 Section 2.3, Table 1. - out->push_back(kInitialByteForDouble); - union { - double from_double; - uint64_t to_uint64; - } reinterpret; - reinterpret.from_double = value; - WriteBytesMostSignificantByteFirst(reinterpret.to_uint64, out); -} - -void EnvelopeEncoder::EncodeStart(std::vector* out) { - assert(byte_size_pos_ == 0); - out->push_back(kInitialByteForEnvelope); - out->push_back(kInitialByteFor32BitLengthByteString); - byte_size_pos_ = out->size(); - out->resize(out->size() + sizeof(uint32_t)); -} - -bool EnvelopeEncoder::EncodeStop(std::vector* out) { - assert(byte_size_pos_ != 0); - // The byte size is the size of the payload, that is, all the - // bytes that were written past the byte size position itself. - uint64_t byte_size = out->size() - (byte_size_pos_ + sizeof(uint32_t)); - // We store exactly 4 bytes, so at most INT32MAX, with most significant - // byte first. - if (byte_size > std::numeric_limits::max()) return false; - for (int shift_bytes = sizeof(uint32_t) - 1; shift_bytes >= 0; - --shift_bytes) { - (*out)[byte_size_pos_++] = 0xff & (byte_size >> (shift_bytes * 8)); - } - return true; -} - -namespace { -class JSONToCBOREncoder : public JSONParserHandler { - public: - JSONToCBOREncoder(std::vector* out, Status* status) - : out_(out), status_(status) { - *status_ = Status(); - } - - void HandleObjectBegin() override { - envelopes_.emplace_back(); - envelopes_.back().EncodeStart(out_); - out_->push_back(kInitialByteIndefiniteLengthMap); - } - - void HandleObjectEnd() override { - out_->push_back(kStopByte); - assert(!envelopes_.empty()); - envelopes_.back().EncodeStop(out_); - envelopes_.pop_back(); - } - - void HandleArrayBegin() override { - envelopes_.emplace_back(); - envelopes_.back().EncodeStart(out_); - out_->push_back(kInitialByteIndefiniteLengthArray); - } - - void HandleArrayEnd() override { - out_->push_back(kStopByte); - assert(!envelopes_.empty()); - envelopes_.back().EncodeStop(out_); - envelopes_.pop_back(); - } - - void HandleString16(std::vector chars) override { - for (uint16_t ch : chars) { - if (ch >= 0x7f) { - // If there's at least one non-7bit character, we encode as UTF16. - EncodeString16(span(chars.data(), chars.size()), out_); - return; - } - } - std::vector sevenbit_chars(chars.begin(), chars.end()); - EncodeString8(span(sevenbit_chars.data(), sevenbit_chars.size()), - out_); - } - - void HandleBinary(std::vector bytes) override { - EncodeBinary(span(bytes.data(), bytes.size()), out_); - } - - void HandleDouble(double value) override { EncodeDouble(value, out_); } - - void HandleInt32(int32_t value) override { EncodeInt32(value, out_); } - - void HandleBool(bool value) override { - // See RFC 7049 Section 2.3, Table 2. - out_->push_back(value ? kEncodedTrue : kEncodedFalse); - } - - void HandleNull() override { - // See RFC 7049 Section 2.3, Table 2. - out_->push_back(kEncodedNull); - } - - void HandleError(Status error) override { - assert(!error.ok()); - *status_ = error; - out_->clear(); - } - - private: - std::vector* out_; - std::vector envelopes_; - Status* status_; -}; -} // namespace - -std::unique_ptr NewJSONToCBOREncoder( - std::vector* out, Status* status) { - return std::unique_ptr(new JSONToCBOREncoder(out, status)); -} - -namespace { -// Below are three parsing routines for CBOR, which cover enough -// to roundtrip JSON messages. -bool ParseMap(int32_t stack_depth, CBORTokenizer* tokenizer, - JSONParserHandler* out); -bool ParseArray(int32_t stack_depth, CBORTokenizer* tokenizer, - JSONParserHandler* out); -bool ParseValue(int32_t stack_depth, CBORTokenizer* tokenizer, - JSONParserHandler* out); - -void ParseUTF16String(CBORTokenizer* tokenizer, JSONParserHandler* out) { - std::vector value; - span rep = tokenizer->GetString16WireRep(); - for (std::ptrdiff_t ii = 0; ii < rep.size(); ii += 2) - value.push_back((rep[ii + 1] << 8) | rep[ii]); - out->HandleString16(std::move(value)); - tokenizer->Next(); -} - -// For now this method only covers US-ASCII. Later, we may allow UTF8. -bool ParseASCIIString(CBORTokenizer* tokenizer, JSONParserHandler* out) { - assert(tokenizer->TokenTag() == CBORTokenTag::STRING8); - std::vector value16; - for (uint8_t ch : tokenizer->GetString8()) { - // We only accept us-ascii (7 bit) strings here. Other strings must - // be encoded with 16 bit (the BYTE_STRING case). - if (ch >= 0x7f) { - out->HandleError( - Status{Error::CBOR_STRING8_MUST_BE_7BIT, tokenizer->Status().pos}); - return false; - } - value16.push_back(ch); - } - out->HandleString16(std::move(value16)); - tokenizer->Next(); - return true; -} - -bool ParseValue(int32_t stack_depth, CBORTokenizer* tokenizer, - JSONParserHandler* out) { - if (stack_depth > kStackLimit) { - out->HandleError( - Status{Error::CBOR_STACK_LIMIT_EXCEEDED, tokenizer->Status().pos}); - return false; - } - // Skip past the envelope to get to what's inside. - if (tokenizer->TokenTag() == CBORTokenTag::ENVELOPE) - tokenizer->EnterEnvelope(); - switch (tokenizer->TokenTag()) { - case CBORTokenTag::ERROR_VALUE: - out->HandleError(tokenizer->Status()); - return false; - case CBORTokenTag::DONE: - out->HandleError(Status{Error::CBOR_UNEXPECTED_EOF_EXPECTED_VALUE, - tokenizer->Status().pos}); - return false; - case CBORTokenTag::TRUE_VALUE: - out->HandleBool(true); - tokenizer->Next(); - return true; - case CBORTokenTag::FALSE_VALUE: - out->HandleBool(false); - tokenizer->Next(); - return true; - case CBORTokenTag::NULL_VALUE: - out->HandleNull(); - tokenizer->Next(); - return true; - case CBORTokenTag::INT32: - out->HandleInt32(tokenizer->GetInt32()); - tokenizer->Next(); - return true; - case CBORTokenTag::DOUBLE: - out->HandleDouble(tokenizer->GetDouble()); - tokenizer->Next(); - return true; - case CBORTokenTag::STRING8: - return ParseASCIIString(tokenizer, out); - case CBORTokenTag::STRING16: - ParseUTF16String(tokenizer, out); - return true; - case CBORTokenTag::BINARY: { - span binary = tokenizer->GetBinary(); - out->HandleBinary(std::vector(binary.begin(), binary.end())); - tokenizer->Next(); - return true; - } - case CBORTokenTag::MAP_START: - return ParseMap(stack_depth + 1, tokenizer, out); - case CBORTokenTag::ARRAY_START: - return ParseArray(stack_depth + 1, tokenizer, out); - default: - out->HandleError( - Status{Error::CBOR_UNSUPPORTED_VALUE, tokenizer->Status().pos}); - return false; - } -} - -// |bytes| must start with the indefinite length array byte, so basically, -// ParseArray may only be called after an indefinite length array has been -// detected. -bool ParseArray(int32_t stack_depth, CBORTokenizer* tokenizer, - JSONParserHandler* out) { - assert(tokenizer->TokenTag() == CBORTokenTag::ARRAY_START); - tokenizer->Next(); - out->HandleArrayBegin(); - while (tokenizer->TokenTag() != CBORTokenTag::STOP) { - if (tokenizer->TokenTag() == CBORTokenTag::DONE) { - out->HandleError( - Status{Error::CBOR_UNEXPECTED_EOF_IN_ARRAY, tokenizer->Status().pos}); - return false; - } - if (tokenizer->TokenTag() == CBORTokenTag::ERROR_VALUE) { - out->HandleError(tokenizer->Status()); - return false; - } - // Parse value. - if (!ParseValue(stack_depth, tokenizer, out)) return false; - } - out->HandleArrayEnd(); - tokenizer->Next(); - return true; -} - -// |bytes| must start with the indefinite length array byte, so basically, -// ParseArray may only be called after an indefinite length array has been -// detected. -bool ParseMap(int32_t stack_depth, CBORTokenizer* tokenizer, - JSONParserHandler* out) { - assert(tokenizer->TokenTag() == CBORTokenTag::MAP_START); - out->HandleObjectBegin(); - tokenizer->Next(); - while (tokenizer->TokenTag() != CBORTokenTag::STOP) { - if (tokenizer->TokenTag() == CBORTokenTag::DONE) { - out->HandleError( - Status{Error::CBOR_UNEXPECTED_EOF_IN_MAP, tokenizer->Status().pos}); - return false; - } - if (tokenizer->TokenTag() == CBORTokenTag::ERROR_VALUE) { - out->HandleError(tokenizer->Status()); - return false; - } - // Parse key. - if (tokenizer->TokenTag() == CBORTokenTag::STRING8) { - if (!ParseASCIIString(tokenizer, out)) return false; - } else if (tokenizer->TokenTag() == CBORTokenTag::STRING16) { - ParseUTF16String(tokenizer, out); - } else { - out->HandleError( - Status{Error::CBOR_INVALID_MAP_KEY, tokenizer->Status().pos}); - return false; - } - // Parse value. - if (!ParseValue(stack_depth, tokenizer, out)) return false; - } - out->HandleObjectEnd(); - tokenizer->Next(); - return true; -} -} // namespace - -void ParseCBOR(span bytes, JSONParserHandler* json_out) { - if (bytes.empty()) { - json_out->HandleError(Status{Error::CBOR_NO_INPUT, 0}); - return; - } - if (bytes[0] != kInitialByteForEnvelope) { - json_out->HandleError(Status{Error::CBOR_INVALID_START_BYTE, 0}); - return; - } - CBORTokenizer tokenizer(bytes); - if (tokenizer.TokenTag() == CBORTokenTag::ERROR_VALUE) { - json_out->HandleError(tokenizer.Status()); - return; - } - // We checked for the envelope start byte above, so the tokenizer - // must agree here, since it's not an error. - assert(tokenizer.TokenTag() == CBORTokenTag::ENVELOPE); - tokenizer.EnterEnvelope(); - if (tokenizer.TokenTag() != CBORTokenTag::MAP_START) { - json_out->HandleError( - Status{Error::CBOR_MAP_START_EXPECTED, tokenizer.Status().pos}); - return; - } - if (!ParseMap(/*stack_depth=*/1, &tokenizer, json_out)) return; - if (tokenizer.TokenTag() == CBORTokenTag::DONE) return; - if (tokenizer.TokenTag() == CBORTokenTag::ERROR_VALUE) { - json_out->HandleError(tokenizer.Status()); - return; - } - json_out->HandleError( - Status{Error::CBOR_TRAILING_JUNK, tokenizer.Status().pos}); -} - -CBORTokenizer::CBORTokenizer(span bytes) : bytes_(bytes) { - ReadNextToken(/*enter_envelope=*/false); -} -CBORTokenizer::~CBORTokenizer() {} - -CBORTokenTag CBORTokenizer::TokenTag() const { return token_tag_; } - -void CBORTokenizer::Next() { - if (token_tag_ == CBORTokenTag::ERROR_VALUE || token_tag_ == CBORTokenTag::DONE) - return; - ReadNextToken(/*enter_envelope=*/false); -} - -void CBORTokenizer::EnterEnvelope() { - assert(token_tag_ == CBORTokenTag::ENVELOPE); - ReadNextToken(/*enter_envelope=*/true); -} - -Status CBORTokenizer::Status() const { return status_; } - -int32_t CBORTokenizer::GetInt32() const { - assert(token_tag_ == CBORTokenTag::INT32); - // The range checks happen in ::ReadNextToken(). - return static_cast( - token_start_type_ == MajorType::UNSIGNED - ? token_start_internal_value_ - : -static_cast(token_start_internal_value_) - 1); -} - -double CBORTokenizer::GetDouble() const { - assert(token_tag_ == CBORTokenTag::DOUBLE); - union { - uint64_t from_uint64; - double to_double; - } reinterpret; - reinterpret.from_uint64 = ReadBytesMostSignificantByteFirst( - bytes_.subspan(status_.pos + 1)); - return reinterpret.to_double; -} - -span CBORTokenizer::GetString8() const { - assert(token_tag_ == CBORTokenTag::STRING8); - auto length = static_cast(token_start_internal_value_); - return bytes_.subspan(status_.pos + (token_byte_length_ - length), length); -} - -span CBORTokenizer::GetString16WireRep() const { - assert(token_tag_ == CBORTokenTag::STRING16); - auto length = static_cast(token_start_internal_value_); - return bytes_.subspan(status_.pos + (token_byte_length_ - length), length); -} - -span CBORTokenizer::GetBinary() const { - assert(token_tag_ == CBORTokenTag::BINARY); - auto length = static_cast(token_start_internal_value_); - return bytes_.subspan(status_.pos + (token_byte_length_ - length), length); -} - -void CBORTokenizer::ReadNextToken(bool enter_envelope) { - if (enter_envelope) { - status_.pos += kEncodedEnvelopeHeaderSize; - } else { - status_.pos = - status_.pos == Status::npos() ? 0 : status_.pos + token_byte_length_; - } - status_.error = Error::OK; - if (status_.pos >= bytes_.size()) { - token_tag_ = CBORTokenTag::DONE; - return; - } - switch (bytes_[status_.pos]) { - case kStopByte: - SetToken(CBORTokenTag::STOP, 1); - return; - case kInitialByteIndefiniteLengthMap: - SetToken(CBORTokenTag::MAP_START, 1); - return; - case kInitialByteIndefiniteLengthArray: - SetToken(CBORTokenTag::ARRAY_START, 1); - return; - case kEncodedTrue: - SetToken(CBORTokenTag::TRUE_VALUE, 1); - return; - case kEncodedFalse: - SetToken(CBORTokenTag::FALSE_VALUE, 1); - return; - case kEncodedNull: - SetToken(CBORTokenTag::NULL_VALUE, 1); - return; - case kExpectedConversionToBase64Tag: { // BINARY - int8_t bytes_read = - ReadTokenStart(bytes_.subspan(status_.pos + 1), &token_start_type_, - &token_start_internal_value_); - int64_t token_byte_length = 1 + bytes_read + token_start_internal_value_; - if (-1 == bytes_read || token_start_type_ != MajorType::BYTE_STRING || - status_.pos + token_byte_length > bytes_.size()) { - SetError(Error::CBOR_INVALID_BINARY); - return; - } - SetToken(CBORTokenTag::BINARY, - static_cast(token_byte_length)); - return; - } - case kInitialByteForDouble: { // DOUBLE - if (status_.pos + kEncodedDoubleSize > bytes_.size()) { - SetError(Error::CBOR_INVALID_DOUBLE); - return; - } - SetToken(CBORTokenTag::DOUBLE, kEncodedDoubleSize); - return; - } - case kInitialByteForEnvelope: { // ENVELOPE - if (status_.pos + kEncodedEnvelopeHeaderSize > bytes_.size()) { - SetError(Error::CBOR_INVALID_ENVELOPE); - return; - } - // The envelope must be a byte string with 32 bit length. - if (bytes_[status_.pos + 1] != kInitialByteFor32BitLengthByteString) { - SetError(Error::CBOR_INVALID_ENVELOPE); - return; - } - // Read the length of the byte string. - token_start_internal_value_ = ReadBytesMostSignificantByteFirst( - bytes_.subspan(status_.pos + 2)); - // Make sure the payload is contained within the message. - if (token_start_internal_value_ + kEncodedEnvelopeHeaderSize + - status_.pos > - static_cast(bytes_.size())) { - SetError(Error::CBOR_INVALID_ENVELOPE); - return; - } - auto length = static_cast(token_start_internal_value_); - SetToken(CBORTokenTag::ENVELOPE, - kEncodedEnvelopeHeaderSize + length); - return; - } - default: { - span remainder = - bytes_.subspan(status_.pos, bytes_.size() - status_.pos); - assert(!remainder.empty()); - int8_t token_start_length = ReadTokenStart(remainder, &token_start_type_, - &token_start_internal_value_); - bool success = token_start_length != -1; - switch (token_start_type_) { - case MajorType::UNSIGNED: // INT32. - if (!success || std::numeric_limits::max() < - token_start_internal_value_) { - SetError(Error::CBOR_INVALID_INT32); - return; - } - SetToken(CBORTokenTag::INT32, token_start_length); - return; - case MajorType::NEGATIVE: // INT32. - if (!success || - std::numeric_limits::min() > - -static_cast(token_start_internal_value_) - 1) { - SetError(Error::CBOR_INVALID_INT32); - return; - } - SetToken(CBORTokenTag::INT32, token_start_length); - return; - case MajorType::STRING: { // STRING8. - if (!success || remainder.size() < static_cast( - token_start_internal_value_)) { - SetError(Error::CBOR_INVALID_STRING8); - return; - } - auto length = static_cast(token_start_internal_value_); - SetToken(CBORTokenTag::STRING8, token_start_length + length); - return; - } - case MajorType::BYTE_STRING: { // STRING16. - if (!success || - remainder.size() < - static_cast(token_start_internal_value_) || - // Must be divisible by 2 since UTF16 is 2 bytes per character. - token_start_internal_value_ & 1) { - SetError(Error::CBOR_INVALID_STRING16); - return; - } - auto length = static_cast(token_start_internal_value_); - SetToken(CBORTokenTag::STRING16, token_start_length + length); - return; - } - case MajorType::ARRAY: - case MajorType::MAP: - case MajorType::TAG: - case MajorType::SIMPLE_VALUE: - SetError(Error::CBOR_UNSUPPORTED_VALUE); - return; - } - } - } -} - -void CBORTokenizer::SetToken(CBORTokenTag token_tag, - std::ptrdiff_t token_byte_length) { - token_tag_ = token_tag; - token_byte_length_ = token_byte_length; -} - -void CBORTokenizer::SetError(Error error) { - token_tag_ = CBORTokenTag::ERROR_VALUE; - status_.error = error; -} - -#if 0 -void DumpCBOR(span cbor) { - std::string indent; - CBORTokenizer tokenizer(cbor); - while (true) { - fprintf(stderr, "%s", indent.c_str()); - switch (tokenizer.TokenTag()) { - case CBORTokenTag::ERROR_VALUE: - fprintf(stderr, "ERROR {status.error=%d, status.pos=%ld}\n", - tokenizer.Status().error, tokenizer.Status().pos); - return; - case CBORTokenTag::DONE: - fprintf(stderr, "DONE\n"); - return; - case CBORTokenTag::TRUE_VALUE: - fprintf(stderr, "TRUE_VALUE\n"); - break; - case CBORTokenTag::FALSE_VALUE: - fprintf(stderr, "FALSE_VALUE\n"); - break; - case CBORTokenTag::NULL_VALUE: - fprintf(stderr, "NULL_VALUE\n"); - break; - case CBORTokenTag::INT32: - fprintf(stderr, "INT32 [%d]\n", tokenizer.GetInt32()); - break; - case CBORTokenTag::DOUBLE: - fprintf(stderr, "DOUBLE [%lf]\n", tokenizer.GetDouble()); - break; - case CBORTokenTag::STRING8: { - span v = tokenizer.GetString8(); - std::string t(v.begin(), v.end()); - fprintf(stderr, "STRING8 [%s]\n", t.c_str()); - break; - } - case CBORTokenTag::STRING16: { - span v = tokenizer.GetString16WireRep(); - std::string t(v.begin(), v.end()); - fprintf(stderr, "STRING16 [%s]\n", t.c_str()); - break; - } - case CBORTokenTag::BINARY: { - span v = tokenizer.GetBinary(); - std::string t(v.begin(), v.end()); - fprintf(stderr, "BINARY [%s]\n", t.c_str()); - break; - } - case CBORTokenTag::MAP_START: - fprintf(stderr, "MAP_START\n"); - indent += " "; - break; - case CBORTokenTag::ARRAY_START: - fprintf(stderr, "ARRAY_START\n"); - indent += " "; - break; - case CBORTokenTag::STOP: - fprintf(stderr, "STOP\n"); - indent.erase(0, 2); - break; - case CBORTokenTag::ENVELOPE: - fprintf(stderr, "ENVELOPE\n"); - tokenizer.EnterEnvelope(); - continue; - } - tokenizer.Next(); - } -} -#endif - - -{% for namespace in config.protocol.namespace %} -} // namespace {{namespace}} -{% endfor %} diff --git a/tools/inspector_protocol/lib/CBOR_h.template b/tools/inspector_protocol/lib/CBOR_h.template deleted file mode 100644 index dd637f19e7d9d9..00000000000000 --- a/tools/inspector_protocol/lib/CBOR_h.template +++ /dev/null @@ -1,416 +0,0 @@ -{# This template is generated by gen_cbor_templates.py. #} -// Generated by lib/CBOR_h.template. - -// Copyright 2019 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef {{"_".join(config.protocol.namespace)}}_CBOR_h -#define {{"_".join(config.protocol.namespace)}}_CBOR_h - -#include -#include -#include -#include - -{% for namespace in config.protocol.namespace %} -namespace {{namespace}} { -{% endfor %} - -// ===== encoding/status.h ===== - -// Error codes. -enum class Error { - OK = 0, - // JSON parsing errors - json_parser.{h,cc}. - JSON_PARSER_UNPROCESSED_INPUT_REMAINS = 0x01, - JSON_PARSER_STACK_LIMIT_EXCEEDED = 0x02, - JSON_PARSER_NO_INPUT = 0x03, - JSON_PARSER_INVALID_TOKEN = 0x04, - JSON_PARSER_INVALID_NUMBER = 0x05, - JSON_PARSER_INVALID_STRING = 0x06, - JSON_PARSER_UNEXPECTED_ARRAY_END = 0x07, - JSON_PARSER_COMMA_OR_ARRAY_END_EXPECTED = 0x08, - JSON_PARSER_STRING_LITERAL_EXPECTED = 0x09, - JSON_PARSER_COLON_EXPECTED = 0x0a, - JSON_PARSER_UNEXPECTED_OBJECT_END = 0x0b, - JSON_PARSER_COMMA_OR_OBJECT_END_EXPECTED = 0x0c, - JSON_PARSER_VALUE_EXPECTED = 0x0d, - - CBOR_INVALID_INT32 = 0x0e, - CBOR_INVALID_DOUBLE = 0x0f, - CBOR_INVALID_ENVELOPE = 0x10, - CBOR_INVALID_STRING8 = 0x11, - CBOR_INVALID_STRING16 = 0x12, - CBOR_INVALID_BINARY = 0x13, - CBOR_UNSUPPORTED_VALUE = 0x14, - CBOR_NO_INPUT = 0x15, - CBOR_INVALID_START_BYTE = 0x16, - CBOR_UNEXPECTED_EOF_EXPECTED_VALUE = 0x17, - CBOR_UNEXPECTED_EOF_IN_ARRAY = 0x18, - CBOR_UNEXPECTED_EOF_IN_MAP = 0x19, - CBOR_INVALID_MAP_KEY = 0x1a, - CBOR_STACK_LIMIT_EXCEEDED = 0x1b, - CBOR_STRING8_MUST_BE_7BIT = 0x1c, - CBOR_TRAILING_JUNK = 0x1d, - CBOR_MAP_START_EXPECTED = 0x1e, -}; - -// A status value with position that can be copied. The default status -// is OK. Usually, error status values should come with a valid position. -struct Status { - static constexpr std::ptrdiff_t npos() { return -1; } - - bool ok() const { return error == Error::OK; } - - Error error = Error::OK; - std::ptrdiff_t pos = npos(); - Status(Error error, std::ptrdiff_t pos) : error(error), pos(pos) {} - Status() = default; -}; - -// ===== encoding/span.h ===== - -// This template is similar to std::span, which will be included in C++20. Like -// std::span it uses ptrdiff_t, which is signed (and thus a bit annoying -// sometimes when comparing with size_t), but other than this it's much simpler. -template -class span { - public: - using index_type = std::ptrdiff_t; - - span() : data_(nullptr), size_(0) {} - span(const T* data, index_type size) : data_(data), size_(size) {} - - const T* data() const { return data_; } - - const T* begin() const { return data_; } - const T* end() const { return data_ + size_; } - - const T& operator[](index_type idx) const { return data_[idx]; } - - span subspan(index_type offset, index_type count) const { - return span(data_ + offset, count); - } - - span subspan(index_type offset) const { - return span(data_ + offset, size_ - offset); - } - - bool empty() const { return size_ == 0; } - - index_type size() const { return size_; } - index_type size_bytes() const { return size_ * sizeof(T); } - - private: - const T* data_; - index_type size_; -}; - -// ===== encoding/json_parser_handler.h ===== - -// Handler interface for JSON parser events. See also json_parser.h. -class JSONParserHandler { - public: - virtual ~JSONParserHandler() = default; - virtual void HandleObjectBegin() = 0; - virtual void HandleObjectEnd() = 0; - virtual void HandleArrayBegin() = 0; - virtual void HandleArrayEnd() = 0; - // TODO(johannes): Support utf8 (requires utf16->utf8 conversion - // internally, including handling mismatched surrogate pairs). - virtual void HandleString16(std::vector chars) = 0; - virtual void HandleBinary(std::vector bytes) = 0; - virtual void HandleDouble(double value) = 0; - virtual void HandleInt32(int32_t value) = 0; - virtual void HandleBool(bool value) = 0; - virtual void HandleNull() = 0; - - // The parser may send one error even after other events have already - // been received. Client code is reponsible to then discard the - // already processed events. - // |error| must be an eror, as in, |error.is_ok()| can't be true. - virtual void HandleError(Status error) = 0; -}; - -// ===== encoding/cbor_internals.h ===== - -namespace cbor { -enum class MajorType; -} - -namespace cbor_internals { - -// Reads the start of a token with definitive size from |bytes|. -// |type| is the major type as specified in RFC 7049 Section 2.1. -// |value| is the payload (e.g. for MajorType::UNSIGNED) or is the size -// (e.g. for BYTE_STRING). -// If successful, returns the number of bytes read. Otherwise returns -1. -int8_t ReadTokenStart(span bytes, cbor::MajorType* type, - uint64_t* value); - -// Writes the start of a token with |type|. The |value| may indicate the size, -// or it may be the payload if the value is an unsigned integer. -void WriteTokenStart(cbor::MajorType type, uint64_t value, - std::vector* encoded); -} // namespace cbor_internals - -// ===== encoding/cbor.h ===== - - -namespace cbor { - -// The major types from RFC 7049 Section 2.1. -enum class MajorType { - UNSIGNED = 0, - NEGATIVE = 1, - BYTE_STRING = 2, - STRING = 3, - ARRAY = 4, - MAP = 5, - TAG = 6, - SIMPLE_VALUE = 7 -}; - -// Indicates the number of bits the "initial byte" needs to be shifted to the -// right after applying |kMajorTypeMask| to produce the major type in the -// lowermost bits. -static constexpr uint8_t kMajorTypeBitShift = 5u; -// Mask selecting the low-order 5 bits of the "initial byte", which is where -// the additional information is encoded. -static constexpr uint8_t kAdditionalInformationMask = 0x1f; -// Mask selecting the high-order 3 bits of the "initial byte", which indicates -// the major type of the encoded value. -static constexpr uint8_t kMajorTypeMask = 0xe0; -// Indicates the integer is in the following byte. -static constexpr uint8_t kAdditionalInformation1Byte = 24u; -// Indicates the integer is in the next 2 bytes. -static constexpr uint8_t kAdditionalInformation2Bytes = 25u; -// Indicates the integer is in the next 4 bytes. -static constexpr uint8_t kAdditionalInformation4Bytes = 26u; -// Indicates the integer is in the next 8 bytes. -static constexpr uint8_t kAdditionalInformation8Bytes = 27u; - -// Encodes the initial byte, consisting of the |type| in the first 3 bits -// followed by 5 bits of |additional_info|. -constexpr uint8_t EncodeInitialByte(MajorType type, uint8_t additional_info) { - return (static_cast(type) << kMajorTypeBitShift) | - (additional_info & kAdditionalInformationMask); -} - -// TAG 24 indicates that what follows is a byte string which is -// encoded in CBOR format. We use this as a wrapper for -// maps and arrays, allowing us to skip them, because the -// byte string carries its size (byte length). -// https://tools.ietf.org/html/rfc7049#section-2.4.4.1 -static constexpr uint8_t kInitialByteForEnvelope = - EncodeInitialByte(MajorType::TAG, 24); -// The initial byte for a byte string with at most 2^32 bytes -// of payload. This is used for envelope encoding, even if -// the byte string is shorter. -static constexpr uint8_t kInitialByteFor32BitLengthByteString = - EncodeInitialByte(MajorType::BYTE_STRING, 26); - -// See RFC 7049 Section 2.2.1, indefinite length arrays / maps have additional -// info = 31. -static constexpr uint8_t kInitialByteIndefiniteLengthArray = - EncodeInitialByte(MajorType::ARRAY, 31); -static constexpr uint8_t kInitialByteIndefiniteLengthMap = - EncodeInitialByte(MajorType::MAP, 31); -// See RFC 7049 Section 2.3, Table 1; this is used for finishing indefinite -// length maps / arrays. -static constexpr uint8_t kStopByte = - EncodeInitialByte(MajorType::SIMPLE_VALUE, 31); - -} // namespace cbor - -// The binary encoding for the inspector protocol follows the CBOR specification -// (RFC 7049). Additional constraints: -// - Only indefinite length maps and arrays are supported. -// - Maps and arrays are wrapped with an envelope, that is, a -// CBOR tag with value 24 followed by a byte string specifying -// the byte length of the enclosed map / array. The byte string -// must use a 32 bit wide length. -// - At the top level, a message must be an indefinite length map -// wrapped by an envelope. -// - Maximal size for messages is 2^32 (4 GB). -// - For scalars, we support only the int32_t range, encoded as -// UNSIGNED/NEGATIVE (major types 0 / 1). -// - UTF16 strings, including with unbalanced surrogate pairs, are encoded -// as CBOR BYTE_STRING (major type 2). For such strings, the number of -// bytes encoded must be even. -// - UTF8 strings (major type 3) may only have ASCII characters -// (7 bit US-ASCII). -// - Arbitrary byte arrays, in the inspector protocol called 'binary', -// are encoded as BYTE_STRING (major type 2), prefixed with a byte -// indicating base64 when rendered as JSON. - -// Encodes |value| as |UNSIGNED| (major type 0) iff >= 0, or |NEGATIVE| -// (major type 1) iff < 0. -void EncodeInt32(int32_t value, std::vector* out); - -// Encodes a UTF16 string as a BYTE_STRING (major type 2). Each utf16 -// character in |in| is emitted with most significant byte first, -// appending to |out|. -void EncodeString16(span in, std::vector* out); - -// Encodes a UTF8 string |in| as STRING (major type 3). -void EncodeString8(span in, std::vector* out); - -// Encodes arbitrary binary data in |in| as a BYTE_STRING (major type 2) with -// definitive length, prefixed with tag 22 indicating expected conversion to -// base64 (see RFC 7049, Table 3 and Section 2.4.4.2). -void EncodeBinary(span in, std::vector* out); - -// Encodes / decodes a double as Major type 7 (SIMPLE_VALUE), -// with additional info = 27, followed by 8 bytes in big endian. -void EncodeDouble(double value, std::vector* out); - -// Some constants for CBOR tokens that only take a single byte on the wire. -uint8_t EncodeTrue(); -uint8_t EncodeFalse(); -uint8_t EncodeNull(); -uint8_t EncodeIndefiniteLengthArrayStart(); -uint8_t EncodeIndefiniteLengthMapStart(); -uint8_t EncodeStop(); - -// An envelope indicates the byte length of a wrapped item. -// We use this for maps and array, which allows the decoder -// to skip such (nested) values whole sale. -// It's implemented as a CBOR tag (major type 6) with additional -// info = 24, followed by a byte string with a 32 bit length value; -// so the maximal structure that we can wrap is 2^32 bits long. -// See also: https://tools.ietf.org/html/rfc7049#section-2.4.4.1 -class EnvelopeEncoder { - public: - // Emits the envelope start bytes and records the position for the - // byte size in |byte_size_pos_|. Also emits empty bytes for the - // byte sisze so that encoding can continue. - void EncodeStart(std::vector* out); - // This records the current size in |out| at position byte_size_pos_. - // Returns true iff successful. - bool EncodeStop(std::vector* out); - - private: - std::size_t byte_size_pos_ = 0; -}; - -// This can be used to convert from JSON to CBOR, by passing the -// return value to the routines in json_parser.h. The handler will encode into -// |out|, and iff an error occurs it will set |status| to an error and clear -// |out|. Otherwise, |status.ok()| will be |true|. -std::unique_ptr NewJSONToCBOREncoder( - std::vector* out, Status* status); - -// Parses a CBOR encoded message from |bytes|, sending JSON events to -// |json_out|. If an error occurs, sends |out->HandleError|, and parsing stops. -// The client is responsible for discarding the already received information in -// that case. -void ParseCBOR(span bytes, JSONParserHandler* json_out); - -// Tags for the tokens within a CBOR message that CBORStream understands. -// Note that this is not the same terminology as the CBOR spec (RFC 7049), -// but rather, our adaptation. For instance, we lump unsigned and signed -// major type into INT32 here (and disallow values outside the int32_t range). -enum class CBORTokenTag { - // Encountered an error in the structure of the message. Consult - // status() for details. - ERROR_VALUE, - // Booleans and NULL. - TRUE_VALUE, - FALSE_VALUE, - NULL_VALUE, - // An int32_t (signed 32 bit integer). - INT32, - // A double (64 bit floating point). - DOUBLE, - // A UTF8 string. - STRING8, - // A UTF16 string. - STRING16, - // A binary string. - BINARY, - // Starts an indefinite length map; after the map start we expect - // alternating keys and values, followed by STOP. - MAP_START, - // Starts an indefinite length array; after the array start we - // expect values, followed by STOP. - ARRAY_START, - // Ends a map or an array. - STOP, - // An envelope indicator, wrapping a map or array. - // Internally this carries the byte length of the wrapped - // map or array. While CBORTokenizer::Next() will read / skip the entire - // envelope, CBORTokenizer::EnterEnvelope() reads the tokens - // inside of it. - ENVELOPE, - // We've reached the end there is nothing else to read. - DONE, -}; - -// CBORTokenizer segments a CBOR message, presenting the tokens therein as -// numbers, strings, etc. This is not a complete CBOR parser, but makes it much -// easier to implement one (e.g. ParseCBOR, above). It can also be used to parse -// messages partially. -class CBORTokenizer { - public: - explicit CBORTokenizer(span bytes); - ~CBORTokenizer(); - - // Identifies the current token that we're looking at, - // or ERROR_VALUE (in which ase ::Status() has details) - // or DONE (if we're past the last token). - CBORTokenTag TokenTag() const; - - // Advances to the next token. - void Next(); - // Can only be called if TokenTag() == CBORTokenTag::ENVELOPE. - // While Next() would skip past the entire envelope / what it's - // wrapping, EnterEnvelope positions the cursor inside of the envelope, - // letting the client explore the nested structure. - void EnterEnvelope(); - - // If TokenTag() is CBORTokenTag::ERROR_VALUE, then Status().error describes - // the error more precisely; otherwise it'll be set to Error::OK. - // In either case, Status().pos is the current position. - struct Status Status() const; - - // The following methods retrieve the token values. They can only - // be called if TokenTag() matches. - - // To be called only if ::TokenTag() == CBORTokenTag::INT32. - int32_t GetInt32() const; - - // To be called only if ::TokenTag() == CBORTokenTag::DOUBLE. - double GetDouble() const; - - // To be called only if ::TokenTag() == CBORTokenTag::STRING8. - span GetString8() const; - - // Wire representation for STRING16 is low byte first (little endian). - // To be called only if ::TokenTag() == CBORTokenTag::STRING16. - span GetString16WireRep() const; - - // To be called only if ::TokenTag() == CBORTokenTag::BINARY. - span GetBinary() const; - - private: - void ReadNextToken(bool enter_envelope); - void SetToken(CBORTokenTag token, std::ptrdiff_t token_byte_length); - void SetError(Error error); - - span bytes_; - CBORTokenTag token_tag_; - struct Status status_; - std::ptrdiff_t token_byte_length_; - cbor::MajorType token_start_type_; - uint64_t token_start_internal_value_; -}; - -void DumpCBOR(span cbor); - - -{% for namespace in config.protocol.namespace %} -} // namespace {{namespace}} -{% endfor %} -#endif // !defined({{"_".join(config.protocol.namespace)}}_CBOR_h) diff --git a/tools/inspector_protocol/lib/Collections_h.template b/tools/inspector_protocol/lib/Collections_h.template deleted file mode 100644 index 7505a17bfa6e68..00000000000000 --- a/tools/inspector_protocol/lib/Collections_h.template +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2016 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef {{"_".join(config.protocol.namespace)}}_Collections_h -#define {{"_".join(config.protocol.namespace)}}_Collections_h - -#include {{format_include(config.protocol.package, "Forward")}} -#include - -#if defined(__APPLE__) && !defined(_LIBCPP_VERSION) -#include -#include - -{% for namespace in config.protocol.namespace %} -namespace {{namespace}} { -{% endfor %} - -template using HashMap = std::map; -template using HashSet = std::set; - -{% for namespace in config.protocol.namespace %} -} // namespace {{namespace}} -{% endfor %} - -#else -#include -#include - -{% for namespace in config.protocol.namespace %} -namespace {{namespace}} { -{% endfor %} - -template using HashMap = std::unordered_map; -template using HashSet = std::unordered_set; - -{% for namespace in config.protocol.namespace %} -} // namespace {{namespace}} -{% endfor %} - -#endif // defined(__APPLE__) && !defined(_LIBCPP_VERSION) - -#endif // !defined({{"_".join(config.protocol.namespace)}}_Collections_h) diff --git a/tools/inspector_protocol/lib/Values_cpp.template b/tools/inspector_protocol/lib/Values_cpp.template index 4b4ba994151db7..764b4d37d9a7d5 100644 --- a/tools/inspector_protocol/lib/Values_cpp.template +++ b/tools/inspector_protocol/lib/Values_cpp.template @@ -66,21 +66,21 @@ static constexpr int kStackLimitValues = 1000; // Below are three parsing routines for CBOR, which cover enough // to roundtrip JSON messages. -std::unique_ptr parseMap(int32_t stack_depth, CBORTokenizer* tokenizer); -std::unique_ptr parseArray(int32_t stack_depth, CBORTokenizer* tokenizer); -std::unique_ptr parseValue(int32_t stack_depth, CBORTokenizer* tokenizer); +std::unique_ptr parseMap(int32_t stack_depth, cbor::CBORTokenizer* tokenizer); +std::unique_ptr parseArray(int32_t stack_depth, cbor::CBORTokenizer* tokenizer); +std::unique_ptr parseValue(int32_t stack_depth, cbor::CBORTokenizer* tokenizer); // |bytes| must start with the indefinite length array byte, so basically, // ParseArray may only be called after an indefinite length array has been // detected. -std::unique_ptr parseArray(int32_t stack_depth, CBORTokenizer* tokenizer) { - DCHECK(tokenizer->TokenTag() == CBORTokenTag::ARRAY_START); +std::unique_ptr parseArray(int32_t stack_depth, cbor::CBORTokenizer* tokenizer) { + DCHECK(tokenizer->TokenTag() == cbor::CBORTokenTag::ARRAY_START); tokenizer->Next(); auto list = ListValue::create(); - while (tokenizer->TokenTag() != CBORTokenTag::STOP) { + while (tokenizer->TokenTag() != cbor::CBORTokenTag::STOP) { // Error::CBOR_UNEXPECTED_EOF_IN_ARRAY - if (tokenizer->TokenTag() == CBORTokenTag::DONE) return nullptr; - if (tokenizer->TokenTag() == CBORTokenTag::ERROR_VALUE) return nullptr; + if (tokenizer->TokenTag() == cbor::CBORTokenTag::DONE) return nullptr; + if (tokenizer->TokenTag() == cbor::CBORTokenTag::ERROR_VALUE) return nullptr; // Parse value. auto value = parseValue(stack_depth, tokenizer); if (!value) return nullptr; @@ -91,60 +91,66 @@ std::unique_ptr parseArray(int32_t stack_depth, CBORTokenizer* tokeni } std::unique_ptr parseValue( - int32_t stack_depth, CBORTokenizer* tokenizer) { + int32_t stack_depth, cbor::CBORTokenizer* tokenizer) { // Error::CBOR_STACK_LIMIT_EXCEEDED if (stack_depth > kStackLimitValues) return nullptr; // Skip past the envelope to get to what's inside. - if (tokenizer->TokenTag() == CBORTokenTag::ENVELOPE) + if (tokenizer->TokenTag() == cbor::CBORTokenTag::ENVELOPE) tokenizer->EnterEnvelope(); switch (tokenizer->TokenTag()) { - case CBORTokenTag::ERROR_VALUE: + case cbor::CBORTokenTag::ERROR_VALUE: return nullptr; - case CBORTokenTag::DONE: + case cbor::CBORTokenTag::DONE: // Error::CBOR_UNEXPECTED_EOF_EXPECTED_VALUE return nullptr; - case CBORTokenTag::TRUE_VALUE: { + case cbor::CBORTokenTag::TRUE_VALUE: { std::unique_ptr value = FundamentalValue::create(true); tokenizer->Next(); return value; } - case CBORTokenTag::FALSE_VALUE: { + case cbor::CBORTokenTag::FALSE_VALUE: { std::unique_ptr value = FundamentalValue::create(false); tokenizer->Next(); return value; } - case CBORTokenTag::NULL_VALUE: { + case cbor::CBORTokenTag::NULL_VALUE: { std::unique_ptr value = FundamentalValue::null(); tokenizer->Next(); return value; } - case CBORTokenTag::INT32: { + case cbor::CBORTokenTag::INT32: { std::unique_ptr value = FundamentalValue::create(tokenizer->GetInt32()); tokenizer->Next(); return value; } - case CBORTokenTag::DOUBLE: { + case cbor::CBORTokenTag::DOUBLE: { std::unique_ptr value = FundamentalValue::create(tokenizer->GetDouble()); tokenizer->Next(); return value; } - case CBORTokenTag::STRING8: { + case cbor::CBORTokenTag::STRING8: { span str = tokenizer->GetString8(); - std::unique_ptr value = StringValue::create(StringUtil::fromUTF8(str.data(), str.size())); + std::unique_ptr value = + StringValue::create(StringUtil::fromUTF8(str.data(), str.size())); tokenizer->Next(); return value; } - case CBORTokenTag::STRING16: - // NOT SUPPORTED YET. - return nullptr; - case CBORTokenTag::BINARY: { + case cbor::CBORTokenTag::STRING16: { + span wire = tokenizer->GetString16WireRep(); + DCHECK_EQ(wire.size() & 1, 0u); + std::unique_ptr value = StringValue::create(StringUtil::fromUTF16( + reinterpret_cast(wire.data()), wire.size() / 2)); + tokenizer->Next(); + return value; + } + case cbor::CBORTokenTag::BINARY: { span payload = tokenizer->GetBinary(); tokenizer->Next(); return BinaryValue::create(Binary::fromSpan(payload.data(), payload.size())); } - case CBORTokenTag::MAP_START: + case cbor::CBORTokenTag::MAP_START: return parseMap(stack_depth + 1, tokenizer); - case CBORTokenTag::ARRAY_START: + case cbor::CBORTokenTag::ARRAY_START: return parseArray(stack_depth + 1, tokenizer); default: // Error::CBOR_UNSUPPORTED_VALUE @@ -156,22 +162,22 @@ std::unique_ptr parseValue( // ParseArray may only be called after an indefinite length array has been // detected. std::unique_ptr parseMap( - int32_t stack_depth, CBORTokenizer* tokenizer) { + int32_t stack_depth, cbor::CBORTokenizer* tokenizer) { auto dict = DictionaryValue::create(); tokenizer->Next(); - while (tokenizer->TokenTag() != CBORTokenTag::STOP) { - if (tokenizer->TokenTag() == CBORTokenTag::DONE) { + while (tokenizer->TokenTag() != cbor::CBORTokenTag::STOP) { + if (tokenizer->TokenTag() == cbor::CBORTokenTag::DONE) { // Error::CBOR_UNEXPECTED_EOF_IN_MAP return nullptr; } - if (tokenizer->TokenTag() == CBORTokenTag::ERROR_VALUE) return nullptr; + if (tokenizer->TokenTag() == cbor::CBORTokenTag::ERROR_VALUE) return nullptr; // Parse key. String key; - if (tokenizer->TokenTag() == CBORTokenTag::STRING8) { + if (tokenizer->TokenTag() == cbor::CBORTokenTag::STRING8) { span key_span = tokenizer->GetString8(); key = StringUtil::fromUTF8(key_span.data(), key_span.size()); tokenizer->Next(); - } else if (tokenizer->TokenTag() == CBORTokenTag::STRING16) { + } else if (tokenizer->TokenTag() == cbor::CBORTokenTag::STRING16) { return nullptr; // STRING16 not supported yet. } else { // Error::CBOR_INVALID_MAP_KEY @@ -196,22 +202,21 @@ std::unique_ptr Value::parseBinary(const uint8_t* data, size_t size) { if (bytes.empty()) return nullptr; // Error::CBOR_INVALID_START_BYTE - // TODO(johannes): EncodeInitialByteForEnvelope() method. - if (bytes[0] != 0xd8) return nullptr; + if (bytes[0] != cbor::InitialByteForEnvelope()) return nullptr; - CBORTokenizer tokenizer(bytes); - if (tokenizer.TokenTag() == CBORTokenTag::ERROR_VALUE) return nullptr; + cbor::CBORTokenizer tokenizer(bytes); + if (tokenizer.TokenTag() == cbor::CBORTokenTag::ERROR_VALUE) return nullptr; // We checked for the envelope start byte above, so the tokenizer // must agree here, since it's not an error. - DCHECK(tokenizer.TokenTag() == CBORTokenTag::ENVELOPE); + DCHECK(tokenizer.TokenTag() == cbor::CBORTokenTag::ENVELOPE); tokenizer.EnterEnvelope(); // Error::MAP_START_EXPECTED - if (tokenizer.TokenTag() != CBORTokenTag::MAP_START) return nullptr; + if (tokenizer.TokenTag() != cbor::CBORTokenTag::MAP_START) return nullptr; std::unique_ptr result = parseMap(/*stack_depth=*/1, &tokenizer); if (!result) return nullptr; - if (tokenizer.TokenTag() == CBORTokenTag::DONE) return result; - if (tokenizer.TokenTag() == CBORTokenTag::ERROR_VALUE) return nullptr; + if (tokenizer.TokenTag() == cbor::CBORTokenTag::DONE) return result; + if (tokenizer.TokenTag() == cbor::CBORTokenTag::ERROR_VALUE) return nullptr; // Error::CBOR_TRAILING_JUNK return nullptr; } @@ -249,7 +254,7 @@ void Value::writeJSON(StringBuilder* output) const void Value::writeBinary(std::vector* bytes) const { DCHECK(m_type == TypeNull); - bytes->push_back(EncodeNull()); + bytes->push_back(cbor::EncodeNull()); } std::unique_ptr Value::clone() const @@ -326,13 +331,13 @@ void FundamentalValue::writeJSON(StringBuilder* output) const void FundamentalValue::writeBinary(std::vector* bytes) const { switch (type()) { case TypeDouble: - EncodeDouble(m_doubleValue, bytes); + cbor::EncodeDouble(m_doubleValue, bytes); return; case TypeInteger: - EncodeInt32(m_integerValue, bytes); + cbor::EncodeInt32(m_integerValue, bytes); return; case TypeBoolean: - bytes->push_back(m_boolValue ? EncodeTrue() : EncodeFalse()); + bytes->push_back(m_boolValue ? cbor::EncodeTrue() : cbor::EncodeFalse()); return; default: DCHECK(false); @@ -363,10 +368,37 @@ void StringValue::writeJSON(StringBuilder* output) const StringUtil::builderAppendQuotedString(*output, m_stringValue); } +namespace { +// This routine distinguishes between the current encoding for a given +// string |s|, and calls encoding routines that will +// - Ensure that all ASCII strings end up being encoded as UTF8 in +// the wire format - e.g., EncodeFromUTF16 will detect ASCII and +// do the (trivial) transcode to STRING8 on the wire, but if it's +// not ASCII it'll do STRING16. +// - Select a format that's cheap to convert to. E.g., we don't +// have LATIN1 on the wire, so we call EncodeFromLatin1 which +// transcodes to UTF8 if needed. +void EncodeString(const String& s, std::vector* out) { + if (StringUtil::CharacterCount(s) == 0) { + cbor::EncodeString8(span(nullptr, 0), out); // Empty string. + } else if (StringUtil::CharactersLatin1(s)) { + cbor::EncodeFromLatin1(span(StringUtil::CharactersLatin1(s), + StringUtil::CharacterCount(s)), + out); + } else if (StringUtil::CharactersUTF16(s)) { + cbor::EncodeFromUTF16(span(StringUtil::CharactersUTF16(s), + StringUtil::CharacterCount(s)), + out); + } else if (StringUtil::CharactersUTF8(s)) { + cbor::EncodeString8(span(StringUtil::CharactersUTF8(s), + StringUtil::CharacterCount(s)), + out); + } +} +} // namespace + void StringValue::writeBinary(std::vector* bytes) const { - StringUTF8Adapter utf8(m_stringValue); - EncodeString8(span(reinterpret_cast(utf8.Data()), - utf8.length()), bytes); + EncodeString(m_stringValue, bytes); } std::unique_ptr StringValue::clone() const @@ -387,7 +419,8 @@ void BinaryValue::writeJSON(StringBuilder* output) const } void BinaryValue::writeBinary(std::vector* bytes) const { - EncodeBinary(span(m_binaryValue.data(), m_binaryValue.size()), bytes); + cbor::EncodeBinary(span(m_binaryValue.data(), + m_binaryValue.size()), bytes); } std::unique_ptr BinaryValue::clone() const @@ -550,19 +583,17 @@ void DictionaryValue::writeJSON(StringBuilder* output) const } void DictionaryValue::writeBinary(std::vector* bytes) const { - EnvelopeEncoder encoder; + cbor::EnvelopeEncoder encoder; encoder.EncodeStart(bytes); - bytes->push_back(EncodeIndefiniteLengthMapStart()); + bytes->push_back(cbor::EncodeIndefiniteLengthMapStart()); for (size_t i = 0; i < m_order.size(); ++i) { const String& key = m_order[i]; Dictionary::const_iterator value = m_data.find(key); DCHECK(value != m_data.cend() && value->second); - StringUTF8Adapter utf8(key); - EncodeString8(span(reinterpret_cast(utf8.Data()), - utf8.length()), bytes); + EncodeString(key, bytes); value->second->writeBinary(bytes); } - bytes->push_back(EncodeStop()); + bytes->push_back(cbor::EncodeStop()); encoder.EncodeStop(bytes); } @@ -601,13 +632,13 @@ void ListValue::writeJSON(StringBuilder* output) const } void ListValue::writeBinary(std::vector* bytes) const { - EnvelopeEncoder encoder; + cbor::EnvelopeEncoder encoder; encoder.EncodeStart(bytes); - bytes->push_back(EncodeIndefiniteLengthArrayStart()); + bytes->push_back(cbor::EncodeIndefiniteLengthArrayStart()); for (size_t i = 0; i < m_data.size(); ++i) { m_data[i]->writeBinary(bytes); } - bytes->push_back(EncodeStop()); + bytes->push_back(cbor::EncodeStop()); encoder.EncodeStop(bytes); } diff --git a/tools/inspector_protocol/lib/base_string_adapter_cc.template b/tools/inspector_protocol/lib/base_string_adapter_cc.template index ed3316446f4a51..639b39bb520d85 100644 --- a/tools/inspector_protocol/lib/base_string_adapter_cc.template +++ b/tools/inspector_protocol/lib/base_string_adapter_cc.template @@ -136,7 +136,7 @@ std::unique_ptr StringUtil::parseMessage( reinterpret_cast(message.data()), message.length()); } - std::unique_ptr value = base::JSONReader::Read(message); + std::unique_ptr value = base::JSONReader::ReadDeprecated(message); return toProtocolValue(value.get(), 1000); } @@ -185,6 +185,13 @@ void StringBuilder::reserveCapacity(size_t capacity) { string_.reserve(capacity); } +// static +String StringUtil::fromUTF16(const uint16_t* data, size_t length) { + std::string utf8; + base::UTF16ToUTF8(reinterpret_cast(data), length, &utf8); + return utf8; +} + Binary::Binary() : bytes_(new base::RefCountedBytes) {} Binary::Binary(const Binary& binary) : bytes_(binary.bytes_) {} Binary::Binary(scoped_refptr bytes) : bytes_(bytes) {} @@ -230,75 +237,6 @@ Binary Binary::fromSpan(const uint8_t* data, size_t size) { new base::RefCountedBytes(data, size))); } -namespace { -int32_t ReadEnvelopeSize(const uint8_t* in) { - return (in[0] << 24) + (in[1] << 16) + (in[2] << 8) + in[3]; -} - -void WriteEnvelopeSize(uint32_t value, uint8_t* out) { - *(out++) = (value >> 24) & 0xFF; - *(out++) = (value >> 16) & 0xFF; - *(out++) = (value >> 8) & 0xFF; - *(out++) = (value) & 0xFF; -} - -} - -bool AppendStringValueToMapBinary(base::StringPiece in, - base::StringPiece key, base::StringPiece value, std::string* out) { - if (in.size() < 1 + 1 + 4 + 1 + 1) - return false; - const uint8_t* envelope = reinterpret_cast(in.data()); - if (cbor::kInitialByteForEnvelope != envelope[0]) - return false; - if (cbor::kInitialByteFor32BitLengthByteString != envelope[1]) - return false; - if (cbor::kInitialByteIndefiniteLengthMap != envelope[6]) - return false; - - uint32_t envelope_size = ReadEnvelopeSize(envelope + 2); - if (envelope_size + 2 + 4 != in.size()) - return false; - if (cbor::kStopByte != static_cast(*in.rbegin())) - return false; - - std::vector encoded_entry; - encoded_entry.reserve(1 + 4 + key.size() + 1 + 4 + value.size()); - span key_span( - reinterpret_cast(key.data()), key.size()); - EncodeString8(key_span, &encoded_entry); - span value_span( - reinterpret_cast(value.data()), value.size()); - EncodeString8(value_span, &encoded_entry); - - out->clear(); - out->reserve(in.size() + encoded_entry.size()); - out->append(in.begin(), in.end() - 1); - out->append(reinterpret_cast(encoded_entry.data()), - encoded_entry.size()); - out->append(1, static_cast(cbor::kStopByte)); - std::size_t new_size = envelope_size + out->size() - in.size(); - if (new_size > static_cast( - std::numeric_limits::max())) { - return false; - } - WriteEnvelopeSize(new_size, reinterpret_cast(&*out->begin() + 2)); - return true; -} - -bool AppendStringValueToMapJSON(base::StringPiece in, - base::StringPiece key, base::StringPiece value, std::string* out) { - if (!in.length() || *in.rbegin() != '}') - return false; - std::string suffix = - base::StringPrintf(", \"%s\": \"%s\"}", key.begin(), value.begin()); - out->clear(); - out->reserve(in.length() + suffix.length() - 1); - out->append(in.data(), in.length() - 1); - out->append(suffix); - return true; -} - {% for namespace in config.protocol.namespace %} } // namespace {{namespace}} {% endfor %} diff --git a/tools/inspector_protocol/lib/base_string_adapter_h.template b/tools/inspector_protocol/lib/base_string_adapter_h.template index b0215e0745017a..8bf3c355c0e584 100644 --- a/tools/inspector_protocol/lib/base_string_adapter_h.template +++ b/tools/inspector_protocol/lib/base_string_adapter_h.template @@ -32,16 +32,6 @@ class Value; using String = std::string; using ProtocolMessage = std::string; -class {{config.lib.export_macro}} StringUTF8Adapter { - public: - StringUTF8Adapter(const std::string& string) : string_(string) { } - const char* Data() const { return string_.data(); } - size_t length() const { return string_.length(); } - - private: - const std::string& string_; -}; - class {{config.lib.export_macro}} StringBuilder { public: StringBuilder(); @@ -109,6 +99,15 @@ class {{config.lib.export_macro}} StringUtil { static String fromUTF8(const uint8_t* data, size_t length) { return std::string(reinterpret_cast(data), length); } + + static String fromUTF16(const uint16_t* data, size_t length); + + static const uint8_t* CharactersLatin1(const String& s) { return nullptr; } + static const uint8_t* CharactersUTF8(const String& s) { + return reinterpret_cast(s.data()); + } + static const uint16_t* CharactersUTF16(const String& s) { return nullptr; } + static size_t CharacterCount(const String& s) { return s.size(); } }; // A read-only sequence of uninterpreted bytes with reference-counted storage. @@ -137,12 +136,6 @@ class {{config.lib.export_macro}} Binary { std::unique_ptr toProtocolValue(const base::Value* value, int depth); std::unique_ptr toBaseValue(Value* value, int depth); - -bool AppendStringValueToMapBinary(base::StringPiece in, - base::StringPiece key, base::StringPiece value, std::string* out); -bool AppendStringValueToMapJSON(base::StringPiece in, - base::StringPiece key, base::StringPiece value, std::string* out); - {% for namespace in config.protocol.namespace %} } // namespace {{namespace}} {% endfor %} diff --git a/tools/inspector_protocol/lib/encoding_cpp.template b/tools/inspector_protocol/lib/encoding_cpp.template new file mode 100644 index 00000000000000..54a46ecd203d4d --- /dev/null +++ b/tools/inspector_protocol/lib/encoding_cpp.template @@ -0,0 +1,2199 @@ +{# This template is generated by gen_cbor_templates.py. #} +// Generated by lib/encoding_cpp.template. + +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + + +#include +#include +#include +#include +#include +#include + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +// ===== encoding/encoding.cc ===== + +// ============================================================================= +// Status and Error codes +// ============================================================================= + +std::string Status::ToASCIIString() const { + switch (error) { + case Error::OK: + return "OK"; + case Error::JSON_PARSER_UNPROCESSED_INPUT_REMAINS: + return ToASCIIString("JSON: unprocessed input remains"); + case Error::JSON_PARSER_STACK_LIMIT_EXCEEDED: + return ToASCIIString("JSON: stack limit exceeded"); + case Error::JSON_PARSER_NO_INPUT: + return ToASCIIString("JSON: no input"); + case Error::JSON_PARSER_INVALID_TOKEN: + return ToASCIIString("JSON: invalid token"); + case Error::JSON_PARSER_INVALID_NUMBER: + return ToASCIIString("JSON: invalid number"); + case Error::JSON_PARSER_INVALID_STRING: + return ToASCIIString("JSON: invalid string"); + case Error::JSON_PARSER_UNEXPECTED_ARRAY_END: + return ToASCIIString("JSON: unexpected array end"); + case Error::JSON_PARSER_COMMA_OR_ARRAY_END_EXPECTED: + return ToASCIIString("JSON: comma or array end expected"); + case Error::JSON_PARSER_STRING_LITERAL_EXPECTED: + return ToASCIIString("JSON: string literal expected"); + case Error::JSON_PARSER_COLON_EXPECTED: + return ToASCIIString("JSON: colon expected"); + case Error::JSON_PARSER_UNEXPECTED_MAP_END: + return ToASCIIString("JSON: unexpected map end"); + case Error::JSON_PARSER_COMMA_OR_MAP_END_EXPECTED: + return ToASCIIString("JSON: comma or map end expected"); + case Error::JSON_PARSER_VALUE_EXPECTED: + return ToASCIIString("JSON: value expected"); + + case Error::CBOR_INVALID_INT32: + return ToASCIIString("CBOR: invalid int32"); + case Error::CBOR_INVALID_DOUBLE: + return ToASCIIString("CBOR: invalid double"); + case Error::CBOR_INVALID_ENVELOPE: + return ToASCIIString("CBOR: invalid envelope"); + case Error::CBOR_INVALID_STRING8: + return ToASCIIString("CBOR: invalid string8"); + case Error::CBOR_INVALID_STRING16: + return ToASCIIString("CBOR: invalid string16"); + case Error::CBOR_INVALID_BINARY: + return ToASCIIString("CBOR: invalid binary"); + case Error::CBOR_UNSUPPORTED_VALUE: + return ToASCIIString("CBOR: unsupported value"); + case Error::CBOR_NO_INPUT: + return ToASCIIString("CBOR: no input"); + case Error::CBOR_INVALID_START_BYTE: + return ToASCIIString("CBOR: invalid start byte"); + case Error::CBOR_UNEXPECTED_EOF_EXPECTED_VALUE: + return ToASCIIString("CBOR: unexpected eof expected value"); + case Error::CBOR_UNEXPECTED_EOF_IN_ARRAY: + return ToASCIIString("CBOR: unexpected eof in array"); + case Error::CBOR_UNEXPECTED_EOF_IN_MAP: + return ToASCIIString("CBOR: unexpected eof in map"); + case Error::CBOR_INVALID_MAP_KEY: + return ToASCIIString("CBOR: invalid map key"); + case Error::CBOR_STACK_LIMIT_EXCEEDED: + return ToASCIIString("CBOR: stack limit exceeded"); + case Error::CBOR_TRAILING_JUNK: + return ToASCIIString("CBOR: trailing junk"); + case Error::CBOR_MAP_START_EXPECTED: + return ToASCIIString("CBOR: map start expected"); + case Error::CBOR_MAP_STOP_EXPECTED: + return ToASCIIString("CBOR: map stop expected"); + case Error::CBOR_ENVELOPE_SIZE_LIMIT_EXCEEDED: + return ToASCIIString("CBOR: envelope size limit exceeded"); + } + // Some compilers can't figure out that we can't get here. + return "INVALID ERROR CODE"; +} + +std::string Status::ToASCIIString(const char* msg) const { + return std::string(msg) + " at position " + std::to_string(pos); +} + +namespace cbor { +namespace { +// Indicates the number of bits the "initial byte" needs to be shifted to the +// right after applying |kMajorTypeMask| to produce the major type in the +// lowermost bits. +static constexpr uint8_t kMajorTypeBitShift = 5u; +// Mask selecting the low-order 5 bits of the "initial byte", which is where +// the additional information is encoded. +static constexpr uint8_t kAdditionalInformationMask = 0x1f; +// Mask selecting the high-order 3 bits of the "initial byte", which indicates +// the major type of the encoded value. +static constexpr uint8_t kMajorTypeMask = 0xe0; +// Indicates the integer is in the following byte. +static constexpr uint8_t kAdditionalInformation1Byte = 24u; +// Indicates the integer is in the next 2 bytes. +static constexpr uint8_t kAdditionalInformation2Bytes = 25u; +// Indicates the integer is in the next 4 bytes. +static constexpr uint8_t kAdditionalInformation4Bytes = 26u; +// Indicates the integer is in the next 8 bytes. +static constexpr uint8_t kAdditionalInformation8Bytes = 27u; + +// Encodes the initial byte, consisting of the |type| in the first 3 bits +// followed by 5 bits of |additional_info|. +constexpr uint8_t EncodeInitialByte(MajorType type, uint8_t additional_info) { + return (static_cast(type) << kMajorTypeBitShift) | + (additional_info & kAdditionalInformationMask); +} + +// TAG 24 indicates that what follows is a byte string which is +// encoded in CBOR format. We use this as a wrapper for +// maps and arrays, allowing us to skip them, because the +// byte string carries its size (byte length). +// https://tools.ietf.org/html/rfc7049#section-2.4.4.1 +static constexpr uint8_t kInitialByteForEnvelope = + EncodeInitialByte(MajorType::TAG, 24); +// The initial byte for a byte string with at most 2^32 bytes +// of payload. This is used for envelope encoding, even if +// the byte string is shorter. +static constexpr uint8_t kInitialByteFor32BitLengthByteString = + EncodeInitialByte(MajorType::BYTE_STRING, 26); + +// See RFC 7049 Section 2.2.1, indefinite length arrays / maps have additional +// info = 31. +static constexpr uint8_t kInitialByteIndefiniteLengthArray = + EncodeInitialByte(MajorType::ARRAY, 31); +static constexpr uint8_t kInitialByteIndefiniteLengthMap = + EncodeInitialByte(MajorType::MAP, 31); +// See RFC 7049 Section 2.3, Table 1; this is used for finishing indefinite +// length maps / arrays. +static constexpr uint8_t kStopByte = + EncodeInitialByte(MajorType::SIMPLE_VALUE, 31); + +// See RFC 7049 Section 2.3, Table 2. +static constexpr uint8_t kEncodedTrue = + EncodeInitialByte(MajorType::SIMPLE_VALUE, 21); +static constexpr uint8_t kEncodedFalse = + EncodeInitialByte(MajorType::SIMPLE_VALUE, 20); +static constexpr uint8_t kEncodedNull = + EncodeInitialByte(MajorType::SIMPLE_VALUE, 22); +static constexpr uint8_t kInitialByteForDouble = + EncodeInitialByte(MajorType::SIMPLE_VALUE, 27); + +// See RFC 7049 Table 3 and Section 2.4.4.2. This is used as a prefix for +// arbitrary binary data encoded as BYTE_STRING. +static constexpr uint8_t kExpectedConversionToBase64Tag = + EncodeInitialByte(MajorType::TAG, 22); + +// Writes the bytes for |v| to |out|, starting with the most significant byte. +// See also: https://commandcenter.blogspot.com/2012/04/byte-order-fallacy.html +template +void WriteBytesMostSignificantByteFirst(T v, C* out) { + for (int shift_bytes = sizeof(T) - 1; shift_bytes >= 0; --shift_bytes) + out->push_back(0xff & (v >> (shift_bytes * 8))); +} + +// Extracts sizeof(T) bytes from |in| to extract a value of type T +// (e.g. uint64_t, uint32_t, ...), most significant byte first. +// See also: https://commandcenter.blogspot.com/2012/04/byte-order-fallacy.html +template +T ReadBytesMostSignificantByteFirst(span in) { + assert(in.size() >= sizeof(T)); + T result = 0; + for (size_t shift_bytes = 0; shift_bytes < sizeof(T); ++shift_bytes) + result |= T(in[sizeof(T) - 1 - shift_bytes]) << (shift_bytes * 8); + return result; +} +} // namespace + +namespace internals { +// Reads the start of a token with definitive size from |bytes|. +// |type| is the major type as specified in RFC 7049 Section 2.1. +// |value| is the payload (e.g. for MajorType::UNSIGNED) or is the size +// (e.g. for BYTE_STRING). +// If successful, returns the number of bytes read. Otherwise returns -1. +// TODO(johannes): change return type to size_t and use 0 for error. +int8_t ReadTokenStart(span bytes, MajorType* type, uint64_t* value) { + if (bytes.empty()) + return -1; + uint8_t initial_byte = bytes[0]; + *type = MajorType((initial_byte & kMajorTypeMask) >> kMajorTypeBitShift); + + uint8_t additional_information = initial_byte & kAdditionalInformationMask; + if (additional_information < 24) { + // Values 0-23 are encoded directly into the additional info of the + // initial byte. + *value = additional_information; + return 1; + } + if (additional_information == kAdditionalInformation1Byte) { + // Values 24-255 are encoded with one initial byte, followed by the value. + if (bytes.size() < 2) + return -1; + *value = ReadBytesMostSignificantByteFirst(bytes.subspan(1)); + return 2; + } + if (additional_information == kAdditionalInformation2Bytes) { + // Values 256-65535: 1 initial byte + 2 bytes payload. + if (bytes.size() < 1 + sizeof(uint16_t)) + return -1; + *value = ReadBytesMostSignificantByteFirst(bytes.subspan(1)); + return 3; + } + if (additional_information == kAdditionalInformation4Bytes) { + // 32 bit uint: 1 initial byte + 4 bytes payload. + if (bytes.size() < 1 + sizeof(uint32_t)) + return -1; + *value = ReadBytesMostSignificantByteFirst(bytes.subspan(1)); + return 5; + } + if (additional_information == kAdditionalInformation8Bytes) { + // 64 bit uint: 1 initial byte + 8 bytes payload. + if (bytes.size() < 1 + sizeof(uint64_t)) + return -1; + *value = ReadBytesMostSignificantByteFirst(bytes.subspan(1)); + return 9; + } + return -1; +} + +// Writes the start of a token with |type|. The |value| may indicate the size, +// or it may be the payload if the value is an unsigned integer. +template +void WriteTokenStartTmpl(MajorType type, uint64_t value, C* encoded) { + if (value < 24) { + // Values 0-23 are encoded directly into the additional info of the + // initial byte. + encoded->push_back(EncodeInitialByte(type, /*additional_info=*/value)); + return; + } + if (value <= std::numeric_limits::max()) { + // Values 24-255 are encoded with one initial byte, followed by the value. + encoded->push_back(EncodeInitialByte(type, kAdditionalInformation1Byte)); + encoded->push_back(value); + return; + } + if (value <= std::numeric_limits::max()) { + // Values 256-65535: 1 initial byte + 2 bytes payload. + encoded->push_back(EncodeInitialByte(type, kAdditionalInformation2Bytes)); + WriteBytesMostSignificantByteFirst(value, encoded); + return; + } + if (value <= std::numeric_limits::max()) { + // 32 bit uint: 1 initial byte + 4 bytes payload. + encoded->push_back(EncodeInitialByte(type, kAdditionalInformation4Bytes)); + WriteBytesMostSignificantByteFirst(static_cast(value), + encoded); + return; + } + // 64 bit uint: 1 initial byte + 8 bytes payload. + encoded->push_back(EncodeInitialByte(type, kAdditionalInformation8Bytes)); + WriteBytesMostSignificantByteFirst(value, encoded); +} +void WriteTokenStart(MajorType type, + uint64_t value, + std::vector* encoded) { + WriteTokenStartTmpl(type, value, encoded); +} +void WriteTokenStart(MajorType type, uint64_t value, std::string* encoded) { + WriteTokenStartTmpl(type, value, encoded); +} +} // namespace internals + +// ============================================================================= +// Detecting CBOR content +// ============================================================================= + +uint8_t InitialByteForEnvelope() { + return kInitialByteForEnvelope; +} +uint8_t InitialByteFor32BitLengthByteString() { + return kInitialByteFor32BitLengthByteString; +} +bool IsCBORMessage(span msg) { + return msg.size() >= 6 && msg[0] == InitialByteForEnvelope() && + msg[1] == InitialByteFor32BitLengthByteString(); +} + +// ============================================================================= +// Encoding invidiual CBOR items +// ============================================================================= + +uint8_t EncodeTrue() { + return kEncodedTrue; +} +uint8_t EncodeFalse() { + return kEncodedFalse; +} +uint8_t EncodeNull() { + return kEncodedNull; +} + +uint8_t EncodeIndefiniteLengthArrayStart() { + return kInitialByteIndefiniteLengthArray; +} + +uint8_t EncodeIndefiniteLengthMapStart() { + return kInitialByteIndefiniteLengthMap; +} + +uint8_t EncodeStop() { + return kStopByte; +} + +template +void EncodeInt32Tmpl(int32_t value, C* out) { + if (value >= 0) { + internals::WriteTokenStart(MajorType::UNSIGNED, value, out); + } else { + uint64_t representation = static_cast(-(value + 1)); + internals::WriteTokenStart(MajorType::NEGATIVE, representation, out); + } +} +void EncodeInt32(int32_t value, std::vector* out) { + EncodeInt32Tmpl(value, out); +} +void EncodeInt32(int32_t value, std::string* out) { + EncodeInt32Tmpl(value, out); +} + +template +void EncodeString16Tmpl(span in, C* out) { + uint64_t byte_length = static_cast(in.size_bytes()); + internals::WriteTokenStart(MajorType::BYTE_STRING, byte_length, out); + // When emitting UTF16 characters, we always write the least significant byte + // first; this is because it's the native representation for X86. + // TODO(johannes): Implement a more efficient thing here later, e.g. + // casting *iff* the machine has this byte order. + // The wire format for UTF16 chars will probably remain the same + // (least significant byte first) since this way we can have + // golden files, unittests, etc. that port easily and universally. + // See also: + // https://commandcenter.blogspot.com/2012/04/byte-order-fallacy.html + for (const uint16_t two_bytes : in) { + out->push_back(two_bytes); + out->push_back(two_bytes >> 8); + } +} +void EncodeString16(span in, std::vector* out) { + EncodeString16Tmpl(in, out); +} +void EncodeString16(span in, std::string* out) { + EncodeString16Tmpl(in, out); +} + +template +void EncodeString8Tmpl(span in, C* out) { + internals::WriteTokenStart(MajorType::STRING, + static_cast(in.size_bytes()), out); + out->insert(out->end(), in.begin(), in.end()); +} +void EncodeString8(span in, std::vector* out) { + EncodeString8Tmpl(in, out); +} +void EncodeString8(span in, std::string* out) { + EncodeString8Tmpl(in, out); +} + +template +void EncodeFromLatin1Tmpl(span latin1, C* out) { + for (size_t ii = 0; ii < latin1.size(); ++ii) { + if (latin1[ii] <= 127) + continue; + // If there's at least one non-ASCII char, convert to UTF8. + std::vector utf8(latin1.begin(), latin1.begin() + ii); + for (; ii < latin1.size(); ++ii) { + if (latin1[ii] <= 127) { + utf8.push_back(latin1[ii]); + } else { + // 0xC0 means it's a UTF8 sequence with 2 bytes. + utf8.push_back((latin1[ii] >> 6) | 0xc0); + utf8.push_back((latin1[ii] | 0x80) & 0xbf); + } + } + EncodeString8(SpanFrom(utf8), out); + return; + } + EncodeString8(latin1, out); +} +void EncodeFromLatin1(span latin1, std::vector* out) { + EncodeFromLatin1Tmpl(latin1, out); +} +void EncodeFromLatin1(span latin1, std::string* out) { + EncodeFromLatin1Tmpl(latin1, out); +} + +template +void EncodeFromUTF16Tmpl(span utf16, C* out) { + // If there's at least one non-ASCII char, encode as STRING16 (UTF16). + for (uint16_t ch : utf16) { + if (ch <= 127) + continue; + EncodeString16(utf16, out); + return; + } + // It's all US-ASCII, strip out every second byte and encode as UTF8. + internals::WriteTokenStart(MajorType::STRING, + static_cast(utf16.size()), out); + out->insert(out->end(), utf16.begin(), utf16.end()); +} +void EncodeFromUTF16(span utf16, std::vector* out) { + EncodeFromUTF16Tmpl(utf16, out); +} +void EncodeFromUTF16(span utf16, std::string* out) { + EncodeFromUTF16Tmpl(utf16, out); +} + +template +void EncodeBinaryTmpl(span in, C* out) { + out->push_back(kExpectedConversionToBase64Tag); + uint64_t byte_length = static_cast(in.size_bytes()); + internals::WriteTokenStart(MajorType::BYTE_STRING, byte_length, out); + out->insert(out->end(), in.begin(), in.end()); +} +void EncodeBinary(span in, std::vector* out) { + EncodeBinaryTmpl(in, out); +} +void EncodeBinary(span in, std::string* out) { + EncodeBinaryTmpl(in, out); +} + +// A double is encoded with a specific initial byte +// (kInitialByteForDouble) plus the 64 bits of payload for its value. +constexpr size_t kEncodedDoubleSize = 1 + sizeof(uint64_t); + +// An envelope is encoded with a specific initial byte +// (kInitialByteForEnvelope), plus the start byte for a BYTE_STRING with a 32 +// bit wide length, plus a 32 bit length for that string. +constexpr size_t kEncodedEnvelopeHeaderSize = 1 + 1 + sizeof(uint32_t); + +template +void EncodeDoubleTmpl(double value, C* out) { + // The additional_info=27 indicates 64 bits for the double follow. + // See RFC 7049 Section 2.3, Table 1. + out->push_back(kInitialByteForDouble); + union { + double from_double; + uint64_t to_uint64; + } reinterpret; + reinterpret.from_double = value; + WriteBytesMostSignificantByteFirst(reinterpret.to_uint64, out); +} +void EncodeDouble(double value, std::vector* out) { + EncodeDoubleTmpl(value, out); +} +void EncodeDouble(double value, std::string* out) { + EncodeDoubleTmpl(value, out); +} + +// ============================================================================= +// cbor::EnvelopeEncoder - for wrapping submessages +// ============================================================================= + +template +void EncodeStartTmpl(C* out, size_t* byte_size_pos) { + assert(*byte_size_pos == 0); + out->push_back(kInitialByteForEnvelope); + out->push_back(kInitialByteFor32BitLengthByteString); + *byte_size_pos = out->size(); + out->resize(out->size() + sizeof(uint32_t)); +} + +void EnvelopeEncoder::EncodeStart(std::vector* out) { + EncodeStartTmpl>(out, &byte_size_pos_); +} + +void EnvelopeEncoder::EncodeStart(std::string* out) { + EncodeStartTmpl(out, &byte_size_pos_); +} + +template +bool EncodeStopTmpl(C* out, size_t* byte_size_pos) { + assert(*byte_size_pos != 0); + // The byte size is the size of the payload, that is, all the + // bytes that were written past the byte size position itself. + uint64_t byte_size = out->size() - (*byte_size_pos + sizeof(uint32_t)); + // We store exactly 4 bytes, so at most INT32MAX, with most significant + // byte first. + if (byte_size > std::numeric_limits::max()) + return false; + for (int shift_bytes = sizeof(uint32_t) - 1; shift_bytes >= 0; + --shift_bytes) { + (*out)[(*byte_size_pos)++] = 0xff & (byte_size >> (shift_bytes * 8)); + } + return true; +} + +bool EnvelopeEncoder::EncodeStop(std::vector* out) { + return EncodeStopTmpl(out, &byte_size_pos_); +} + +bool EnvelopeEncoder::EncodeStop(std::string* out) { + return EncodeStopTmpl(out, &byte_size_pos_); +} + +// ============================================================================= +// cbor::NewCBOREncoder - for encoding from a streaming parser +// ============================================================================= + +namespace { +template +class CBOREncoder : public StreamingParserHandler { + public: + CBOREncoder(C* out, Status* status) : out_(out), status_(status) { + *status_ = Status(); + } + + void HandleMapBegin() override { + if (!status_->ok()) + return; + envelopes_.emplace_back(); + envelopes_.back().EncodeStart(out_); + out_->push_back(kInitialByteIndefiniteLengthMap); + } + + void HandleMapEnd() override { + if (!status_->ok()) + return; + out_->push_back(kStopByte); + assert(!envelopes_.empty()); + if (!envelopes_.back().EncodeStop(out_)) { + HandleError( + Status(Error::CBOR_ENVELOPE_SIZE_LIMIT_EXCEEDED, out_->size())); + return; + } + envelopes_.pop_back(); + } + + void HandleArrayBegin() override { + if (!status_->ok()) + return; + envelopes_.emplace_back(); + envelopes_.back().EncodeStart(out_); + out_->push_back(kInitialByteIndefiniteLengthArray); + } + + void HandleArrayEnd() override { + if (!status_->ok()) + return; + out_->push_back(kStopByte); + assert(!envelopes_.empty()); + if (!envelopes_.back().EncodeStop(out_)) { + HandleError( + Status(Error::CBOR_ENVELOPE_SIZE_LIMIT_EXCEEDED, out_->size())); + return; + } + envelopes_.pop_back(); + } + + void HandleString8(span chars) override { + if (!status_->ok()) + return; + EncodeString8(chars, out_); + } + + void HandleString16(span chars) override { + if (!status_->ok()) + return; + EncodeFromUTF16(chars, out_); + } + + void HandleBinary(span bytes) override { + if (!status_->ok()) + return; + EncodeBinary(bytes, out_); + } + + void HandleDouble(double value) override { + if (!status_->ok()) + return; + EncodeDouble(value, out_); + } + + void HandleInt32(int32_t value) override { + if (!status_->ok()) + return; + EncodeInt32(value, out_); + } + + void HandleBool(bool value) override { + if (!status_->ok()) + return; + // See RFC 7049 Section 2.3, Table 2. + out_->push_back(value ? kEncodedTrue : kEncodedFalse); + } + + void HandleNull() override { + if (!status_->ok()) + return; + // See RFC 7049 Section 2.3, Table 2. + out_->push_back(kEncodedNull); + } + + void HandleError(Status error) override { + if (!status_->ok()) + return; + *status_ = error; + out_->clear(); + } + + private: + C* out_; + std::vector envelopes_; + Status* status_; +}; +} // namespace + +std::unique_ptr NewCBOREncoder( + std::vector* out, + Status* status) { + return std::unique_ptr( + new CBOREncoder>(out, status)); +} +std::unique_ptr NewCBOREncoder(std::string* out, + Status* status) { + return std::unique_ptr( + new CBOREncoder(out, status)); +} + +// ============================================================================= +// cbor::CBORTokenizer - for parsing individual CBOR items +// ============================================================================= + +CBORTokenizer::CBORTokenizer(span bytes) : bytes_(bytes) { + ReadNextToken(/*enter_envelope=*/false); +} +CBORTokenizer::~CBORTokenizer() {} + +CBORTokenTag CBORTokenizer::TokenTag() const { + return token_tag_; +} + +void CBORTokenizer::Next() { + if (token_tag_ == CBORTokenTag::ERROR_VALUE || + token_tag_ == CBORTokenTag::DONE) + return; + ReadNextToken(/*enter_envelope=*/false); +} + +void CBORTokenizer::EnterEnvelope() { + assert(token_tag_ == CBORTokenTag::ENVELOPE); + ReadNextToken(/*enter_envelope=*/true); +} + +Status CBORTokenizer::Status() const { + return status_; +} + +// The following accessor functions ::GetInt32, ::GetDouble, +// ::GetString8, ::GetString16WireRep, ::GetBinary, ::GetEnvelopeContents +// assume that a particular token was recognized in ::ReadNextToken. +// That's where all the error checking is done. By design, +// the accessors (assuming the token was recognized) never produce +// an error. + +int32_t CBORTokenizer::GetInt32() const { + assert(token_tag_ == CBORTokenTag::INT32); + // The range checks happen in ::ReadNextToken(). + return static_cast( + token_start_type_ == MajorType::UNSIGNED + ? token_start_internal_value_ + : -static_cast(token_start_internal_value_) - 1); +} + +double CBORTokenizer::GetDouble() const { + assert(token_tag_ == CBORTokenTag::DOUBLE); + union { + uint64_t from_uint64; + double to_double; + } reinterpret; + reinterpret.from_uint64 = ReadBytesMostSignificantByteFirst( + bytes_.subspan(status_.pos + 1)); + return reinterpret.to_double; +} + +span CBORTokenizer::GetString8() const { + assert(token_tag_ == CBORTokenTag::STRING8); + auto length = static_cast(token_start_internal_value_); + return bytes_.subspan(status_.pos + (token_byte_length_ - length), length); +} + +span CBORTokenizer::GetString16WireRep() const { + assert(token_tag_ == CBORTokenTag::STRING16); + auto length = static_cast(token_start_internal_value_); + return bytes_.subspan(status_.pos + (token_byte_length_ - length), length); +} + +span CBORTokenizer::GetBinary() const { + assert(token_tag_ == CBORTokenTag::BINARY); + auto length = static_cast(token_start_internal_value_); + return bytes_.subspan(status_.pos + (token_byte_length_ - length), length); +} + +span CBORTokenizer::GetEnvelopeContents() const { + assert(token_tag_ == CBORTokenTag::ENVELOPE); + auto length = static_cast(token_start_internal_value_); + return bytes_.subspan(status_.pos + kEncodedEnvelopeHeaderSize, length); +} + +// All error checking happens in ::ReadNextToken, so that the accessors +// can avoid having to carry an error return value. +// +// With respect to checking the encoded lengths of strings, arrays, etc: +// On the wire, CBOR uses 1,2,4, and 8 byte unsigned integers, so +// we initially read them as uint64_t, usually into token_start_internal_value_. +// +// However, since these containers have a representation on the machine, +// we need to do corresponding size computations on the input byte array, +// output span (e.g. the payload for a string), etc., and size_t is +// machine specific (in practice either 32 bit or 64 bit). +// +// Further, we must avoid overflowing size_t. Therefore, we use this +// kMaxValidLength constant to: +// - Reject values that are larger than the architecture specific +// max size_t (differs between 32 bit and 64 bit arch). +// - Reserve at least one bit so that we can check against overflows +// when adding lengths (array / string length / etc.); we do this by +// ensuring that the inputs to an addition are <= kMaxValidLength, +// and then checking whether the sum went past it. +// +// See also +// https://chromium.googlesource.com/chromium/src/+/master/docs/security/integer-semantics.md +static const uint64_t kMaxValidLength = + std::min(std::numeric_limits::max() >> 2, + std::numeric_limits::max()); + +void CBORTokenizer::ReadNextToken(bool enter_envelope) { + if (enter_envelope) { + status_.pos += kEncodedEnvelopeHeaderSize; + } else { + status_.pos = + status_.pos == Status::npos() ? 0 : status_.pos + token_byte_length_; + } + status_.error = Error::OK; + if (status_.pos >= bytes_.size()) { + token_tag_ = CBORTokenTag::DONE; + return; + } + const size_t remaining_bytes = bytes_.size() - status_.pos; + switch (bytes_[status_.pos]) { + case kStopByte: + SetToken(CBORTokenTag::STOP, 1); + return; + case kInitialByteIndefiniteLengthMap: + SetToken(CBORTokenTag::MAP_START, 1); + return; + case kInitialByteIndefiniteLengthArray: + SetToken(CBORTokenTag::ARRAY_START, 1); + return; + case kEncodedTrue: + SetToken(CBORTokenTag::TRUE_VALUE, 1); + return; + case kEncodedFalse: + SetToken(CBORTokenTag::FALSE_VALUE, 1); + return; + case kEncodedNull: + SetToken(CBORTokenTag::NULL_VALUE, 1); + return; + case kExpectedConversionToBase64Tag: { // BINARY + const int8_t bytes_read = internals::ReadTokenStart( + bytes_.subspan(status_.pos + 1), &token_start_type_, + &token_start_internal_value_); + if (bytes_read < 0 || token_start_type_ != MajorType::BYTE_STRING || + token_start_internal_value_ > kMaxValidLength) { + SetError(Error::CBOR_INVALID_BINARY); + return; + } + const uint64_t token_byte_length = token_start_internal_value_ + + /* tag before token start: */ 1 + + /* token start: */ bytes_read; + if (token_byte_length > remaining_bytes) { + SetError(Error::CBOR_INVALID_BINARY); + return; + } + SetToken(CBORTokenTag::BINARY, static_cast(token_byte_length)); + return; + } + case kInitialByteForDouble: { // DOUBLE + if (kEncodedDoubleSize > remaining_bytes) { + SetError(Error::CBOR_INVALID_DOUBLE); + return; + } + SetToken(CBORTokenTag::DOUBLE, kEncodedDoubleSize); + return; + } + case kInitialByteForEnvelope: { // ENVELOPE + if (kEncodedEnvelopeHeaderSize > remaining_bytes) { + SetError(Error::CBOR_INVALID_ENVELOPE); + return; + } + // The envelope must be a byte string with 32 bit length. + if (bytes_[status_.pos + 1] != kInitialByteFor32BitLengthByteString) { + SetError(Error::CBOR_INVALID_ENVELOPE); + return; + } + // Read the length of the byte string. + token_start_internal_value_ = ReadBytesMostSignificantByteFirst( + bytes_.subspan(status_.pos + 2)); + if (token_start_internal_value_ > kMaxValidLength) { + SetError(Error::CBOR_INVALID_ENVELOPE); + return; + } + uint64_t token_byte_length = + token_start_internal_value_ + kEncodedEnvelopeHeaderSize; + if (token_byte_length > remaining_bytes) { + SetError(Error::CBOR_INVALID_ENVELOPE); + return; + } + SetToken(CBORTokenTag::ENVELOPE, static_cast(token_byte_length)); + return; + } + default: { + const int8_t token_start_length = internals::ReadTokenStart( + bytes_.subspan(status_.pos), &token_start_type_, + &token_start_internal_value_); + const bool success = token_start_length >= 0; + switch (token_start_type_) { + case MajorType::UNSIGNED: // INT32. + // INT32 is a signed int32 (int32 makes sense for the + // inspector_protocol, it's not a CBOR limitation), so we check + // against the signed max, so that the allowable values are + // 0, 1, 2, ... 2^31 - 1. + if (!success || std::numeric_limits::max() < + token_start_internal_value_) { + SetError(Error::CBOR_INVALID_INT32); + return; + } + SetToken(CBORTokenTag::INT32, token_start_length); + return; + case MajorType::NEGATIVE: { // INT32. + // INT32 is a signed int32 (int32 makes sense for the + // inspector_protocol, it's not a CBOR limitation); in CBOR, + // the negative values for INT32 are represented as NEGATIVE, + // that is, -1 INT32 is represented as 1 << 5 | 0 (major type 1, + // additional info value 0). So here, we compute the INT32 value + // and then check it against the INT32 min. + int64_t actual_value = + -static_cast(token_start_internal_value_) - 1; + if (!success || actual_value < std::numeric_limits::min()) { + SetError(Error::CBOR_INVALID_INT32); + return; + } + SetToken(CBORTokenTag::INT32, token_start_length); + return; + } + case MajorType::STRING: { // STRING8. + if (!success || token_start_internal_value_ > kMaxValidLength) { + SetError(Error::CBOR_INVALID_STRING8); + return; + } + uint64_t token_byte_length = + token_start_internal_value_ + token_start_length; + if (token_byte_length > remaining_bytes) { + SetError(Error::CBOR_INVALID_STRING8); + return; + } + SetToken(CBORTokenTag::STRING8, + static_cast(token_byte_length)); + return; + } + case MajorType::BYTE_STRING: { // STRING16. + // Length must be divisible by 2 since UTF16 is 2 bytes per + // character, hence the &1 check. + if (!success || token_start_internal_value_ > kMaxValidLength || + token_start_internal_value_ & 1) { + SetError(Error::CBOR_INVALID_STRING16); + return; + } + uint64_t token_byte_length = + token_start_internal_value_ + token_start_length; + if (token_byte_length > remaining_bytes) { + SetError(Error::CBOR_INVALID_STRING16); + return; + } + SetToken(CBORTokenTag::STRING16, + static_cast(token_byte_length)); + return; + } + case MajorType::ARRAY: + case MajorType::MAP: + case MajorType::TAG: + case MajorType::SIMPLE_VALUE: + SetError(Error::CBOR_UNSUPPORTED_VALUE); + return; + } + } + } +} + +void CBORTokenizer::SetToken(CBORTokenTag token_tag, size_t token_byte_length) { + token_tag_ = token_tag; + token_byte_length_ = token_byte_length; +} + +void CBORTokenizer::SetError(Error error) { + token_tag_ = CBORTokenTag::ERROR_VALUE; + status_.error = error; +} + +// ============================================================================= +// cbor::ParseCBOR - for receiving streaming parser events for CBOR messages +// ============================================================================= + +namespace { +// When parsing CBOR, we limit recursion depth for objects and arrays +// to this constant. +static constexpr int kStackLimit = 300; + +// Below are three parsing routines for CBOR, which cover enough +// to roundtrip JSON messages. +bool ParseMap(int32_t stack_depth, + CBORTokenizer* tokenizer, + StreamingParserHandler* out); +bool ParseArray(int32_t stack_depth, + CBORTokenizer* tokenizer, + StreamingParserHandler* out); +bool ParseValue(int32_t stack_depth, + CBORTokenizer* tokenizer, + StreamingParserHandler* out); + +void ParseUTF16String(CBORTokenizer* tokenizer, StreamingParserHandler* out) { + std::vector value; + span rep = tokenizer->GetString16WireRep(); + for (size_t ii = 0; ii < rep.size(); ii += 2) + value.push_back((rep[ii + 1] << 8) | rep[ii]); + out->HandleString16(span(value.data(), value.size())); + tokenizer->Next(); +} + +bool ParseUTF8String(CBORTokenizer* tokenizer, StreamingParserHandler* out) { + assert(tokenizer->TokenTag() == CBORTokenTag::STRING8); + out->HandleString8(tokenizer->GetString8()); + tokenizer->Next(); + return true; +} + +bool ParseValue(int32_t stack_depth, + CBORTokenizer* tokenizer, + StreamingParserHandler* out) { + if (stack_depth > kStackLimit) { + out->HandleError( + Status{Error::CBOR_STACK_LIMIT_EXCEEDED, tokenizer->Status().pos}); + return false; + } + // Skip past the envelope to get to what's inside. + if (tokenizer->TokenTag() == CBORTokenTag::ENVELOPE) + tokenizer->EnterEnvelope(); + switch (tokenizer->TokenTag()) { + case CBORTokenTag::ERROR_VALUE: + out->HandleError(tokenizer->Status()); + return false; + case CBORTokenTag::DONE: + out->HandleError(Status{Error::CBOR_UNEXPECTED_EOF_EXPECTED_VALUE, + tokenizer->Status().pos}); + return false; + case CBORTokenTag::TRUE_VALUE: + out->HandleBool(true); + tokenizer->Next(); + return true; + case CBORTokenTag::FALSE_VALUE: + out->HandleBool(false); + tokenizer->Next(); + return true; + case CBORTokenTag::NULL_VALUE: + out->HandleNull(); + tokenizer->Next(); + return true; + case CBORTokenTag::INT32: + out->HandleInt32(tokenizer->GetInt32()); + tokenizer->Next(); + return true; + case CBORTokenTag::DOUBLE: + out->HandleDouble(tokenizer->GetDouble()); + tokenizer->Next(); + return true; + case CBORTokenTag::STRING8: + return ParseUTF8String(tokenizer, out); + case CBORTokenTag::STRING16: + ParseUTF16String(tokenizer, out); + return true; + case CBORTokenTag::BINARY: { + out->HandleBinary(tokenizer->GetBinary()); + tokenizer->Next(); + return true; + } + case CBORTokenTag::MAP_START: + return ParseMap(stack_depth + 1, tokenizer, out); + case CBORTokenTag::ARRAY_START: + return ParseArray(stack_depth + 1, tokenizer, out); + default: + out->HandleError( + Status{Error::CBOR_UNSUPPORTED_VALUE, tokenizer->Status().pos}); + return false; + } +} + +// |bytes| must start with the indefinite length array byte, so basically, +// ParseArray may only be called after an indefinite length array has been +// detected. +bool ParseArray(int32_t stack_depth, + CBORTokenizer* tokenizer, + StreamingParserHandler* out) { + assert(tokenizer->TokenTag() == CBORTokenTag::ARRAY_START); + tokenizer->Next(); + out->HandleArrayBegin(); + while (tokenizer->TokenTag() != CBORTokenTag::STOP) { + if (tokenizer->TokenTag() == CBORTokenTag::DONE) { + out->HandleError( + Status{Error::CBOR_UNEXPECTED_EOF_IN_ARRAY, tokenizer->Status().pos}); + return false; + } + if (tokenizer->TokenTag() == CBORTokenTag::ERROR_VALUE) { + out->HandleError(tokenizer->Status()); + return false; + } + // Parse value. + if (!ParseValue(stack_depth, tokenizer, out)) + return false; + } + out->HandleArrayEnd(); + tokenizer->Next(); + return true; +} + +// |bytes| must start with the indefinite length array byte, so basically, +// ParseArray may only be called after an indefinite length array has been +// detected. +bool ParseMap(int32_t stack_depth, + CBORTokenizer* tokenizer, + StreamingParserHandler* out) { + assert(tokenizer->TokenTag() == CBORTokenTag::MAP_START); + out->HandleMapBegin(); + tokenizer->Next(); + while (tokenizer->TokenTag() != CBORTokenTag::STOP) { + if (tokenizer->TokenTag() == CBORTokenTag::DONE) { + out->HandleError( + Status{Error::CBOR_UNEXPECTED_EOF_IN_MAP, tokenizer->Status().pos}); + return false; + } + if (tokenizer->TokenTag() == CBORTokenTag::ERROR_VALUE) { + out->HandleError(tokenizer->Status()); + return false; + } + // Parse key. + if (tokenizer->TokenTag() == CBORTokenTag::STRING8) { + if (!ParseUTF8String(tokenizer, out)) + return false; + } else if (tokenizer->TokenTag() == CBORTokenTag::STRING16) { + ParseUTF16String(tokenizer, out); + } else { + out->HandleError( + Status{Error::CBOR_INVALID_MAP_KEY, tokenizer->Status().pos}); + return false; + } + // Parse value. + if (!ParseValue(stack_depth, tokenizer, out)) + return false; + } + out->HandleMapEnd(); + tokenizer->Next(); + return true; +} +} // namespace + +void ParseCBOR(span bytes, StreamingParserHandler* out) { + if (bytes.empty()) { + out->HandleError(Status{Error::CBOR_NO_INPUT, 0}); + return; + } + if (bytes[0] != kInitialByteForEnvelope) { + out->HandleError(Status{Error::CBOR_INVALID_START_BYTE, 0}); + return; + } + CBORTokenizer tokenizer(bytes); + if (tokenizer.TokenTag() == CBORTokenTag::ERROR_VALUE) { + out->HandleError(tokenizer.Status()); + return; + } + // We checked for the envelope start byte above, so the tokenizer + // must agree here, since it's not an error. + assert(tokenizer.TokenTag() == CBORTokenTag::ENVELOPE); + tokenizer.EnterEnvelope(); + if (tokenizer.TokenTag() != CBORTokenTag::MAP_START) { + out->HandleError( + Status{Error::CBOR_MAP_START_EXPECTED, tokenizer.Status().pos}); + return; + } + if (!ParseMap(/*stack_depth=*/1, &tokenizer, out)) + return; + if (tokenizer.TokenTag() == CBORTokenTag::DONE) + return; + if (tokenizer.TokenTag() == CBORTokenTag::ERROR_VALUE) { + out->HandleError(tokenizer.Status()); + return; + } + out->HandleError(Status{Error::CBOR_TRAILING_JUNK, tokenizer.Status().pos}); +} + +// ============================================================================= +// cbor::AppendString8EntryToMap - for limited in-place editing of messages +// ============================================================================= + +template +Status AppendString8EntryToCBORMapTmpl(span string8_key, + span string8_value, + C* cbor) { + // Careful below: Don't compare (*cbor)[idx] with a uint8_t, since + // it could be a char (signed!). Instead, use bytes. + span bytes(reinterpret_cast(cbor->data()), + cbor->size()); + CBORTokenizer tokenizer(bytes); + if (tokenizer.TokenTag() == CBORTokenTag::ERROR_VALUE) + return tokenizer.Status(); + if (tokenizer.TokenTag() != CBORTokenTag::ENVELOPE) + return Status(Error::CBOR_INVALID_ENVELOPE, 0); + size_t envelope_size = tokenizer.GetEnvelopeContents().size(); + size_t old_size = cbor->size(); + if (old_size != envelope_size + kEncodedEnvelopeHeaderSize) + return Status(Error::CBOR_INVALID_ENVELOPE, 0); + if (envelope_size == 0 || + (tokenizer.GetEnvelopeContents()[0] != EncodeIndefiniteLengthMapStart())) + return Status(Error::CBOR_MAP_START_EXPECTED, kEncodedEnvelopeHeaderSize); + if (bytes[bytes.size() - 1] != EncodeStop()) + return Status(Error::CBOR_MAP_STOP_EXPECTED, cbor->size() - 1); + cbor->pop_back(); + EncodeString8(string8_key, cbor); + EncodeString8(string8_value, cbor); + cbor->push_back(EncodeStop()); + size_t new_envelope_size = envelope_size + (cbor->size() - old_size); + if (new_envelope_size > std::numeric_limits::max()) + return Status(Error::CBOR_ENVELOPE_SIZE_LIMIT_EXCEEDED, 0); + size_t size_pos = cbor->size() - new_envelope_size - sizeof(uint32_t); + uint8_t* out = reinterpret_cast(&cbor->at(size_pos)); + *(out++) = (new_envelope_size >> 24) & 0xff; + *(out++) = (new_envelope_size >> 16) & 0xff; + *(out++) = (new_envelope_size >> 8) & 0xff; + *(out) = new_envelope_size & 0xff; + return Status(); +} +Status AppendString8EntryToCBORMap(span string8_key, + span string8_value, + std::vector* cbor) { + return AppendString8EntryToCBORMapTmpl(string8_key, string8_value, cbor); +} +Status AppendString8EntryToCBORMap(span string8_key, + span string8_value, + std::string* cbor) { + return AppendString8EntryToCBORMapTmpl(string8_key, string8_value, cbor); +} +} // namespace cbor + +namespace json { + +// ============================================================================= +// json::NewJSONEncoder - for encoding streaming parser events as JSON +// ============================================================================= + +namespace { +// Prints |value| to |out| with 4 hex digits, most significant chunk first. +template +void PrintHex(uint16_t value, C* out) { + for (int ii = 3; ii >= 0; --ii) { + int four_bits = 0xf & (value >> (4 * ii)); + out->push_back(four_bits + ((four_bits <= 9) ? '0' : ('a' - 10))); + } +} + +// In the writer below, we maintain a stack of State instances. +// It is just enough to emit the appropriate delimiters and brackets +// in JSON. +enum class Container { + // Used for the top-level, initial state. + NONE, + // Inside a JSON object. + MAP, + // Inside a JSON array. + ARRAY +}; +class State { + public: + explicit State(Container container) : container_(container) {} + void StartElement(std::vector* out) { StartElementTmpl(out); } + void StartElement(std::string* out) { StartElementTmpl(out); } + Container container() const { return container_; } + + private: + template + void StartElementTmpl(C* out) { + assert(container_ != Container::NONE || size_ == 0); + if (size_ != 0) { + char delim = (!(size_ & 1) || container_ == Container::ARRAY) ? ',' : ':'; + out->push_back(delim); + } + ++size_; + } + + Container container_ = Container::NONE; + int size_ = 0; +}; + +constexpr char kBase64Table[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz0123456789+/"; + +template +void Base64Encode(const span& in, C* out) { + // The following three cases are based on the tables in the example + // section in https://en.wikipedia.org/wiki/Base64. We process three + // input bytes at a time, emitting 4 output bytes at a time. + size_t ii = 0; + + // While possible, process three input bytes. + for (; ii + 3 <= in.size(); ii += 3) { + uint32_t twentyfour_bits = (in[ii] << 16) | (in[ii + 1] << 8) | in[ii + 2]; + out->push_back(kBase64Table[(twentyfour_bits >> 18)]); + out->push_back(kBase64Table[(twentyfour_bits >> 12) & 0x3f]); + out->push_back(kBase64Table[(twentyfour_bits >> 6) & 0x3f]); + out->push_back(kBase64Table[twentyfour_bits & 0x3f]); + } + if (ii + 2 <= in.size()) { // Process two input bytes. + uint32_t twentyfour_bits = (in[ii] << 16) | (in[ii + 1] << 8); + out->push_back(kBase64Table[(twentyfour_bits >> 18)]); + out->push_back(kBase64Table[(twentyfour_bits >> 12) & 0x3f]); + out->push_back(kBase64Table[(twentyfour_bits >> 6) & 0x3f]); + out->push_back('='); // Emit padding. + return; + } + if (ii + 1 <= in.size()) { // Process a single input byte. + uint32_t twentyfour_bits = (in[ii] << 16); + out->push_back(kBase64Table[(twentyfour_bits >> 18)]); + out->push_back(kBase64Table[(twentyfour_bits >> 12) & 0x3f]); + out->push_back('='); // Emit padding. + out->push_back('='); // Emit padding. + } +} + +// Implements a handler for JSON parser events to emit a JSON string. +template +class JSONEncoder : public StreamingParserHandler { + public: + JSONEncoder(const Platform* platform, C* out, Status* status) + : platform_(platform), out_(out), status_(status) { + *status_ = Status(); + state_.emplace(Container::NONE); + } + + void HandleMapBegin() override { + if (!status_->ok()) + return; + assert(!state_.empty()); + state_.top().StartElement(out_); + state_.emplace(Container::MAP); + Emit('{'); + } + + void HandleMapEnd() override { + if (!status_->ok()) + return; + assert(state_.size() >= 2 && state_.top().container() == Container::MAP); + state_.pop(); + Emit('}'); + } + + void HandleArrayBegin() override { + if (!status_->ok()) + return; + state_.top().StartElement(out_); + state_.emplace(Container::ARRAY); + Emit('['); + } + + void HandleArrayEnd() override { + if (!status_->ok()) + return; + assert(state_.size() >= 2 && state_.top().container() == Container::ARRAY); + state_.pop(); + Emit(']'); + } + + void HandleString16(span chars) override { + if (!status_->ok()) + return; + state_.top().StartElement(out_); + Emit('"'); + for (const uint16_t ch : chars) { + if (ch == '"') { + Emit("\\\""); + } else if (ch == '\\') { + Emit("\\\\"); + } else if (ch == '\b') { + Emit("\\b"); + } else if (ch == '\f') { + Emit("\\f"); + } else if (ch == '\n') { + Emit("\\n"); + } else if (ch == '\r') { + Emit("\\r"); + } else if (ch == '\t') { + Emit("\\t"); + } else if (ch >= 32 && ch <= 126) { + Emit(ch); + } else { + Emit("\\u"); + PrintHex(ch, out_); + } + } + Emit('"'); + } + + void HandleString8(span chars) override { + if (!status_->ok()) + return; + state_.top().StartElement(out_); + Emit('"'); + for (size_t ii = 0; ii < chars.size(); ++ii) { + uint8_t c = chars[ii]; + if (c == '"') { + Emit("\\\""); + } else if (c == '\\') { + Emit("\\\\"); + } else if (c == '\b') { + Emit("\\b"); + } else if (c == '\f') { + Emit("\\f"); + } else if (c == '\n') { + Emit("\\n"); + } else if (c == '\r') { + Emit("\\r"); + } else if (c == '\t') { + Emit("\\t"); + } else if (c >= 32 && c <= 126) { + Emit(c); + } else if (c < 32) { + Emit("\\u"); + PrintHex(static_cast(c), out_); + } else { + // Inspect the leading byte to figure out how long the utf8 + // byte sequence is; while doing this initialize |codepoint| + // with the first few bits. + // See table in: https://en.wikipedia.org/wiki/UTF-8 + // byte one is 110x xxxx -> 2 byte utf8 sequence + // byte one is 1110 xxxx -> 3 byte utf8 sequence + // byte one is 1111 0xxx -> 4 byte utf8 sequence + uint32_t codepoint; + int num_bytes_left; + if ((c & 0xe0) == 0xc0) { // 2 byte utf8 sequence + num_bytes_left = 1; + codepoint = c & 0x1f; + } else if ((c & 0xf0) == 0xe0) { // 3 byte utf8 sequence + num_bytes_left = 2; + codepoint = c & 0x0f; + } else if ((c & 0xf8) == 0xf0) { // 4 byte utf8 sequence + codepoint = c & 0x07; + num_bytes_left = 3; + } else { + continue; // invalid leading byte + } + + // If we have enough bytes in our input, decode the remaining ones + // belonging to this Unicode character into |codepoint|. + if (ii + num_bytes_left > chars.size()) + continue; + while (num_bytes_left > 0) { + c = chars[++ii]; + --num_bytes_left; + // Check the next byte is a continuation byte, that is 10xx xxxx. + if ((c & 0xc0) != 0x80) + continue; + codepoint = (codepoint << 6) | (c & 0x3f); + } + + // Disallow overlong encodings for ascii characters, as these + // would include " and other characters significant to JSON + // string termination / control. + if (codepoint < 0x7f) + continue; + // Invalid in UTF8, and can't be represented in UTF16 anyway. + if (codepoint > 0x10ffff) + continue; + + // So, now we transcode to UTF16, + // using the math described at https://en.wikipedia.org/wiki/UTF-16, + // for either one or two 16 bit characters. + if (codepoint < 0xffff) { + Emit("\\u"); + PrintHex(static_cast(codepoint), out_); + continue; + } + codepoint -= 0x10000; + // high surrogate + Emit("\\u"); + PrintHex(static_cast((codepoint >> 10) + 0xd800), out_); + // low surrogate + Emit("\\u"); + PrintHex(static_cast((codepoint & 0x3ff) + 0xdc00), out_); + } + } + Emit('"'); + } + + void HandleBinary(span bytes) override { + if (!status_->ok()) + return; + state_.top().StartElement(out_); + Emit('"'); + Base64Encode(bytes, out_); + Emit('"'); + } + + void HandleDouble(double value) override { + if (!status_->ok()) + return; + state_.top().StartElement(out_); + // JSON cannot represent NaN or Infinity. So, for compatibility, + // we behave like the JSON object in web browsers: emit 'null'. + if (!std::isfinite(value)) { + Emit("null"); + return; + } + std::unique_ptr str_value = platform_->DToStr(value); + + // DToStr may fail to emit a 0 before the decimal dot. E.g. this is + // the case in base::NumberToString in Chromium (which is based on + // dmg_fp). So, much like + // https://cs.chromium.org/chromium/src/base/json/json_writer.cc + // we probe for this and emit the leading 0 anyway if necessary. + const char* chars = str_value.get(); + if (chars[0] == '.') { + Emit('0'); + } else if (chars[0] == '-' && chars[1] == '.') { + Emit("-0"); + ++chars; + } + Emit(chars); + } + + void HandleInt32(int32_t value) override { + if (!status_->ok()) + return; + state_.top().StartElement(out_); + Emit(std::to_string(value)); + } + + void HandleBool(bool value) override { + if (!status_->ok()) + return; + state_.top().StartElement(out_); + Emit(value ? "true" : "false"); + } + + void HandleNull() override { + if (!status_->ok()) + return; + state_.top().StartElement(out_); + Emit("null"); + } + + void HandleError(Status error) override { + assert(!error.ok()); + *status_ = error; + out_->clear(); + } + + private: + void Emit(char c) { out_->push_back(c); } + void Emit(const char* str) { + out_->insert(out_->end(), str, str + strlen(str)); + } + void Emit(const std::string& str) { + out_->insert(out_->end(), str.begin(), str.end()); + } + + const Platform* platform_; + C* out_; + Status* status_; + std::stack state_; +}; +} // namespace + +std::unique_ptr NewJSONEncoder( + const Platform* platform, + std::vector* out, + Status* status) { + return std::unique_ptr( + new JSONEncoder>(platform, out, status)); +} +std::unique_ptr NewJSONEncoder(const Platform* platform, + std::string* out, + Status* status) { + return std::unique_ptr( + new JSONEncoder(platform, out, status)); +} + +// ============================================================================= +// json::ParseJSON - for receiving streaming parser events for JSON. +// ============================================================================= + +namespace { +const int kStackLimit = 300; + +enum Token { + ObjectBegin, + ObjectEnd, + ArrayBegin, + ArrayEnd, + StringLiteral, + Number, + BoolTrue, + BoolFalse, + NullToken, + ListSeparator, + ObjectPairSeparator, + InvalidToken, + NoInput +}; + +const char* const kNullString = "null"; +const char* const kTrueString = "true"; +const char* const kFalseString = "false"; + +template +class JsonParser { + public: + JsonParser(const Platform* platform, StreamingParserHandler* handler) + : platform_(platform), handler_(handler) {} + + void Parse(const Char* start, size_t length) { + start_pos_ = start; + const Char* end = start + length; + const Char* tokenEnd = nullptr; + ParseValue(start, end, &tokenEnd, 0); + if (error_) + return; + if (tokenEnd != end) { + HandleError(Error::JSON_PARSER_UNPROCESSED_INPUT_REMAINS, tokenEnd); + } + } + + private: + bool CharsToDouble(const uint16_t* chars, size_t length, double* result) { + std::string buffer; + buffer.reserve(length + 1); + for (size_t ii = 0; ii < length; ++ii) { + bool is_ascii = !(chars[ii] & ~0x7F); + if (!is_ascii) + return false; + buffer.push_back(static_cast(chars[ii])); + } + return platform_->StrToD(buffer.c_str(), result); + } + + bool CharsToDouble(const uint8_t* chars, size_t length, double* result) { + std::string buffer(reinterpret_cast(chars), length); + return platform_->StrToD(buffer.c_str(), result); + } + + static bool ParseConstToken(const Char* start, + const Char* end, + const Char** token_end, + const char* token) { + // |token| is \0 terminated, it's one of the constants at top of the file. + while (start < end && *token != '\0' && *start++ == *token++) { + } + if (*token != '\0') + return false; + *token_end = start; + return true; + } + + static bool ReadInt(const Char* start, + const Char* end, + const Char** token_end, + bool allow_leading_zeros) { + if (start == end) + return false; + bool has_leading_zero = '0' == *start; + int length = 0; + while (start < end && '0' <= *start && *start <= '9') { + ++start; + ++length; + } + if (!length) + return false; + if (!allow_leading_zeros && length > 1 && has_leading_zero) + return false; + *token_end = start; + return true; + } + + static bool ParseNumberToken(const Char* start, + const Char* end, + const Char** token_end) { + // We just grab the number here. We validate the size in DecodeNumber. + // According to RFC4627, a valid number is: [minus] int [frac] [exp] + if (start == end) + return false; + Char c = *start; + if ('-' == c) + ++start; + + if (!ReadInt(start, end, &start, /*allow_leading_zeros=*/false)) + return false; + if (start == end) { + *token_end = start; + return true; + } + + // Optional fraction part + c = *start; + if ('.' == c) { + ++start; + if (!ReadInt(start, end, &start, /*allow_leading_zeros=*/true)) + return false; + if (start == end) { + *token_end = start; + return true; + } + c = *start; + } + + // Optional exponent part + if ('e' == c || 'E' == c) { + ++start; + if (start == end) + return false; + c = *start; + if ('-' == c || '+' == c) { + ++start; + if (start == end) + return false; + } + if (!ReadInt(start, end, &start, /*allow_leading_zeros=*/true)) + return false; + } + + *token_end = start; + return true; + } + + static bool ReadHexDigits(const Char* start, + const Char* end, + const Char** token_end, + int digits) { + if (end - start < digits) + return false; + for (int i = 0; i < digits; ++i) { + Char c = *start++; + if (!(('0' <= c && c <= '9') || ('a' <= c && c <= 'f') || + ('A' <= c && c <= 'F'))) + return false; + } + *token_end = start; + return true; + } + + static bool ParseStringToken(const Char* start, + const Char* end, + const Char** token_end) { + while (start < end) { + Char c = *start++; + if ('\\' == c) { + if (start == end) + return false; + c = *start++; + // Make sure the escaped char is valid. + switch (c) { + case 'x': + if (!ReadHexDigits(start, end, &start, 2)) + return false; + break; + case 'u': + if (!ReadHexDigits(start, end, &start, 4)) + return false; + break; + case '\\': + case '/': + case 'b': + case 'f': + case 'n': + case 'r': + case 't': + case 'v': + case '"': + break; + default: + return false; + } + } else if ('"' == c) { + *token_end = start; + return true; + } + } + return false; + } + + static bool SkipComment(const Char* start, + const Char* end, + const Char** comment_end) { + if (start == end) + return false; + + if (*start != '/' || start + 1 >= end) + return false; + ++start; + + if (*start == '/') { + // Single line comment, read to newline. + for (++start; start < end; ++start) { + if (*start == '\n' || *start == '\r') { + *comment_end = start + 1; + return true; + } + } + *comment_end = end; + // Comment reaches end-of-input, which is fine. + return true; + } + + if (*start == '*') { + Char previous = '\0'; + // Block comment, read until end marker. + for (++start; start < end; previous = *start++) { + if (previous == '*' && *start == '/') { + *comment_end = start + 1; + return true; + } + } + // Block comment must close before end-of-input. + return false; + } + + return false; + } + + static bool IsSpaceOrNewLine(Char c) { + // \v = vertial tab; \f = form feed page break. + return c == ' ' || c == '\n' || c == '\v' || c == '\f' || c == '\r' || + c == '\t'; + } + + static void SkipWhitespaceAndComments(const Char* start, + const Char* end, + const Char** whitespace_end) { + while (start < end) { + if (IsSpaceOrNewLine(*start)) { + ++start; + } else if (*start == '/') { + const Char* comment_end = nullptr; + if (!SkipComment(start, end, &comment_end)) + break; + start = comment_end; + } else { + break; + } + } + *whitespace_end = start; + } + + static Token ParseToken(const Char* start, + const Char* end, + const Char** tokenStart, + const Char** token_end) { + SkipWhitespaceAndComments(start, end, tokenStart); + start = *tokenStart; + + if (start == end) + return NoInput; + + switch (*start) { + case 'n': + if (ParseConstToken(start, end, token_end, kNullString)) + return NullToken; + break; + case 't': + if (ParseConstToken(start, end, token_end, kTrueString)) + return BoolTrue; + break; + case 'f': + if (ParseConstToken(start, end, token_end, kFalseString)) + return BoolFalse; + break; + case '[': + *token_end = start + 1; + return ArrayBegin; + case ']': + *token_end = start + 1; + return ArrayEnd; + case ',': + *token_end = start + 1; + return ListSeparator; + case '{': + *token_end = start + 1; + return ObjectBegin; + case '}': + *token_end = start + 1; + return ObjectEnd; + case ':': + *token_end = start + 1; + return ObjectPairSeparator; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case '-': + if (ParseNumberToken(start, end, token_end)) + return Number; + break; + case '"': + if (ParseStringToken(start + 1, end, token_end)) + return StringLiteral; + break; + } + return InvalidToken; + } + + static int HexToInt(Char c) { + if ('0' <= c && c <= '9') + return c - '0'; + if ('A' <= c && c <= 'F') + return c - 'A' + 10; + if ('a' <= c && c <= 'f') + return c - 'a' + 10; + assert(false); // Unreachable. + return 0; + } + + static bool DecodeString(const Char* start, + const Char* end, + std::vector* output) { + if (start == end) + return true; + if (start > end) + return false; + output->reserve(end - start); + while (start < end) { + uint16_t c = *start++; + // If the |Char| we're dealing with is really a byte, then + // we have utf8 here, and we need to check for multibyte characters + // and transcode them to utf16 (either one or two utf16 chars). + if (sizeof(Char) == sizeof(uint8_t) && c >= 0x7f) { + // Inspect the leading byte to figure out how long the utf8 + // byte sequence is; while doing this initialize |codepoint| + // with the first few bits. + // See table in: https://en.wikipedia.org/wiki/UTF-8 + // byte one is 110x xxxx -> 2 byte utf8 sequence + // byte one is 1110 xxxx -> 3 byte utf8 sequence + // byte one is 1111 0xxx -> 4 byte utf8 sequence + uint32_t codepoint; + int num_bytes_left; + if ((c & 0xe0) == 0xc0) { // 2 byte utf8 sequence + num_bytes_left = 1; + codepoint = c & 0x1f; + } else if ((c & 0xf0) == 0xe0) { // 3 byte utf8 sequence + num_bytes_left = 2; + codepoint = c & 0x0f; + } else if ((c & 0xf8) == 0xf0) { // 4 byte utf8 sequence + codepoint = c & 0x07; + num_bytes_left = 3; + } else { + return false; // invalid leading byte + } + + // If we have enough bytes in our inpput, decode the remaining ones + // belonging to this Unicode character into |codepoint|. + if (start + num_bytes_left > end) + return false; + while (num_bytes_left > 0) { + c = *start++; + --num_bytes_left; + // Check the next byte is a continuation byte, that is 10xx xxxx. + if ((c & 0xc0) != 0x80) + return false; + codepoint = (codepoint << 6) | (c & 0x3f); + } + + // Disallow overlong encodings for ascii characters, as these + // would include " and other characters significant to JSON + // string termination / control. + if (codepoint < 0x7f) + return false; + // Invalid in UTF8, and can't be represented in UTF16 anyway. + if (codepoint > 0x10ffff) + return false; + + // So, now we transcode to UTF16, + // using the math described at https://en.wikipedia.org/wiki/UTF-16, + // for either one or two 16 bit characters. + if (codepoint < 0xffff) { + output->push_back(codepoint); + continue; + } + codepoint -= 0x10000; + output->push_back((codepoint >> 10) + 0xd800); // high surrogate + output->push_back((codepoint & 0x3ff) + 0xdc00); // low surrogate + continue; + } + if ('\\' != c) { + output->push_back(c); + continue; + } + if (start == end) + return false; + c = *start++; + + if (c == 'x') { + // \x is not supported. + return false; + } + + switch (c) { + case '"': + case '/': + case '\\': + break; + case 'b': + c = '\b'; + break; + case 'f': + c = '\f'; + break; + case 'n': + c = '\n'; + break; + case 'r': + c = '\r'; + break; + case 't': + c = '\t'; + break; + case 'v': + c = '\v'; + break; + case 'u': + c = (HexToInt(*start) << 12) + (HexToInt(*(start + 1)) << 8) + + (HexToInt(*(start + 2)) << 4) + HexToInt(*(start + 3)); + start += 4; + break; + default: + return false; + } + output->push_back(c); + } + return true; + } + + void ParseValue(const Char* start, + const Char* end, + const Char** value_token_end, + int depth) { + if (depth > kStackLimit) { + HandleError(Error::JSON_PARSER_STACK_LIMIT_EXCEEDED, start); + return; + } + const Char* token_start = nullptr; + const Char* token_end = nullptr; + Token token = ParseToken(start, end, &token_start, &token_end); + switch (token) { + case NoInput: + HandleError(Error::JSON_PARSER_NO_INPUT, token_start); + return; + case InvalidToken: + HandleError(Error::JSON_PARSER_INVALID_TOKEN, token_start); + return; + case NullToken: + handler_->HandleNull(); + break; + case BoolTrue: + handler_->HandleBool(true); + break; + case BoolFalse: + handler_->HandleBool(false); + break; + case Number: { + double value; + if (!CharsToDouble(token_start, token_end - token_start, &value)) { + HandleError(Error::JSON_PARSER_INVALID_NUMBER, token_start); + return; + } + if (value >= std::numeric_limits::min() && + value <= std::numeric_limits::max() && + static_cast(value) == value) + handler_->HandleInt32(static_cast(value)); + else + handler_->HandleDouble(value); + break; + } + case StringLiteral: { + std::vector value; + bool ok = DecodeString(token_start + 1, token_end - 1, &value); + if (!ok) { + HandleError(Error::JSON_PARSER_INVALID_STRING, token_start); + return; + } + handler_->HandleString16(span(value.data(), value.size())); + break; + } + case ArrayBegin: { + handler_->HandleArrayBegin(); + start = token_end; + token = ParseToken(start, end, &token_start, &token_end); + while (token != ArrayEnd) { + ParseValue(start, end, &token_end, depth + 1); + if (error_) + return; + + // After a list value, we expect a comma or the end of the list. + start = token_end; + token = ParseToken(start, end, &token_start, &token_end); + if (token == ListSeparator) { + start = token_end; + token = ParseToken(start, end, &token_start, &token_end); + if (token == ArrayEnd) { + HandleError(Error::JSON_PARSER_UNEXPECTED_ARRAY_END, token_start); + return; + } + } else if (token != ArrayEnd) { + // Unexpected value after list value. Bail out. + HandleError(Error::JSON_PARSER_COMMA_OR_ARRAY_END_EXPECTED, + token_start); + return; + } + } + handler_->HandleArrayEnd(); + break; + } + case ObjectBegin: { + handler_->HandleMapBegin(); + start = token_end; + token = ParseToken(start, end, &token_start, &token_end); + while (token != ObjectEnd) { + if (token != StringLiteral) { + HandleError(Error::JSON_PARSER_STRING_LITERAL_EXPECTED, + token_start); + return; + } + std::vector key; + if (!DecodeString(token_start + 1, token_end - 1, &key)) { + HandleError(Error::JSON_PARSER_INVALID_STRING, token_start); + return; + } + handler_->HandleString16(span(key.data(), key.size())); + start = token_end; + + token = ParseToken(start, end, &token_start, &token_end); + if (token != ObjectPairSeparator) { + HandleError(Error::JSON_PARSER_COLON_EXPECTED, token_start); + return; + } + start = token_end; + + ParseValue(start, end, &token_end, depth + 1); + if (error_) + return; + start = token_end; + + // After a key/value pair, we expect a comma or the end of the + // object. + token = ParseToken(start, end, &token_start, &token_end); + if (token == ListSeparator) { + start = token_end; + token = ParseToken(start, end, &token_start, &token_end); + if (token == ObjectEnd) { + HandleError(Error::JSON_PARSER_UNEXPECTED_MAP_END, token_start); + return; + } + } else if (token != ObjectEnd) { + // Unexpected value after last object value. Bail out. + HandleError(Error::JSON_PARSER_COMMA_OR_MAP_END_EXPECTED, + token_start); + return; + } + } + handler_->HandleMapEnd(); + break; + } + + default: + // We got a token that's not a value. + HandleError(Error::JSON_PARSER_VALUE_EXPECTED, token_start); + return; + } + + SkipWhitespaceAndComments(token_end, end, value_token_end); + } + + void HandleError(Error error, const Char* pos) { + assert(error != Error::OK); + if (!error_) { + handler_->HandleError( + Status{error, static_cast(pos - start_pos_)}); + error_ = true; + } + } + + const Char* start_pos_ = nullptr; + bool error_ = false; + const Platform* platform_; + StreamingParserHandler* handler_; +}; +} // namespace + +void ParseJSON(const Platform& platform, + span chars, + StreamingParserHandler* handler) { + JsonParser parser(&platform, handler); + parser.Parse(chars.data(), chars.size()); +} + +void ParseJSON(const Platform& platform, + span chars, + StreamingParserHandler* handler) { + JsonParser parser(&platform, handler); + parser.Parse(chars.data(), chars.size()); +} + +// ============================================================================= +// json::ConvertCBORToJSON, json::ConvertJSONToCBOR - for transcoding +// ============================================================================= +template +Status ConvertCBORToJSONTmpl(const Platform& platform, + span cbor, + C* json) { + Status status; + std::unique_ptr json_writer = + NewJSONEncoder(&platform, json, &status); + cbor::ParseCBOR(cbor, json_writer.get()); + return status; +} + +Status ConvertCBORToJSON(const Platform& platform, + span cbor, + std::vector* json) { + return ConvertCBORToJSONTmpl(platform, cbor, json); +} +Status ConvertCBORToJSON(const Platform& platform, + span cbor, + std::string* json) { + return ConvertCBORToJSONTmpl(platform, cbor, json); +} + +template +Status ConvertJSONToCBORTmpl(const Platform& platform, span json, C* cbor) { + Status status; + std::unique_ptr encoder = + cbor::NewCBOREncoder(cbor, &status); + ParseJSON(platform, json, encoder.get()); + return status; +} +Status ConvertJSONToCBOR(const Platform& platform, + span json, + std::string* cbor) { + return ConvertJSONToCBORTmpl(platform, json, cbor); +} +Status ConvertJSONToCBOR(const Platform& platform, + span json, + std::string* cbor) { + return ConvertJSONToCBORTmpl(platform, json, cbor); +} +Status ConvertJSONToCBOR(const Platform& platform, + span json, + std::vector* cbor) { + return ConvertJSONToCBORTmpl(platform, json, cbor); +} +Status ConvertJSONToCBOR(const Platform& platform, + span json, + std::vector* cbor) { + return ConvertJSONToCBORTmpl(platform, json, cbor); +} +} // namespace json + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} diff --git a/tools/inspector_protocol/lib/encoding_h.template b/tools/inspector_protocol/lib/encoding_h.template new file mode 100644 index 00000000000000..f1a52a1958a14d --- /dev/null +++ b/tools/inspector_protocol/lib/encoding_h.template @@ -0,0 +1,520 @@ +{# This template is generated by gen_cbor_templates.py. #} +// Generated by lib/encoding_h.template. + +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef {{"_".join(config.protocol.namespace)}}_encoding_h +#define {{"_".join(config.protocol.namespace)}}_encoding_h + +#include +#include +#include +#include +#include +#include +#include + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +// ===== encoding/encoding.h ===== + + +// ============================================================================= +// span - sequence of bytes +// ============================================================================= + +// This template is similar to std::span, which will be included in C++20. +template +class span { + public: + using index_type = size_t; + + span() : data_(nullptr), size_(0) {} + span(const T* data, index_type size) : data_(data), size_(size) {} + + const T* data() const { return data_; } + + const T* begin() const { return data_; } + const T* end() const { return data_ + size_; } + + const T& operator[](index_type idx) const { return data_[idx]; } + + span subspan(index_type offset, index_type count) const { + return span(data_ + offset, count); + } + + span subspan(index_type offset) const { + return span(data_ + offset, size_ - offset); + } + + bool empty() const { return size_ == 0; } + + index_type size() const { return size_; } + index_type size_bytes() const { return size_ * sizeof(T); } + + private: + const T* data_; + index_type size_; +}; + +template +span SpanFrom(const std::vector& v) { + return span(v.data(), v.size()); +} + +template +span SpanFrom(const char (&str)[N]) { + return span(reinterpret_cast(str), N - 1); +} + +inline span SpanFrom(const char* str) { + return str ? span(reinterpret_cast(str), strlen(str)) + : span(); +} + +inline span SpanFrom(const std::string& v) { + return span(reinterpret_cast(v.data()), v.size()); +} + +// ============================================================================= +// Status and Error codes +// ============================================================================= +enum class Error { + OK = 0, + // JSON parsing errors - json_parser.{h,cc}. + JSON_PARSER_UNPROCESSED_INPUT_REMAINS = 0x01, + JSON_PARSER_STACK_LIMIT_EXCEEDED = 0x02, + JSON_PARSER_NO_INPUT = 0x03, + JSON_PARSER_INVALID_TOKEN = 0x04, + JSON_PARSER_INVALID_NUMBER = 0x05, + JSON_PARSER_INVALID_STRING = 0x06, + JSON_PARSER_UNEXPECTED_ARRAY_END = 0x07, + JSON_PARSER_COMMA_OR_ARRAY_END_EXPECTED = 0x08, + JSON_PARSER_STRING_LITERAL_EXPECTED = 0x09, + JSON_PARSER_COLON_EXPECTED = 0x0a, + JSON_PARSER_UNEXPECTED_MAP_END = 0x0b, + JSON_PARSER_COMMA_OR_MAP_END_EXPECTED = 0x0c, + JSON_PARSER_VALUE_EXPECTED = 0x0d, + + CBOR_INVALID_INT32 = 0x0e, + CBOR_INVALID_DOUBLE = 0x0f, + CBOR_INVALID_ENVELOPE = 0x10, + CBOR_INVALID_STRING8 = 0x11, + CBOR_INVALID_STRING16 = 0x12, + CBOR_INVALID_BINARY = 0x13, + CBOR_UNSUPPORTED_VALUE = 0x14, + CBOR_NO_INPUT = 0x15, + CBOR_INVALID_START_BYTE = 0x16, + CBOR_UNEXPECTED_EOF_EXPECTED_VALUE = 0x17, + CBOR_UNEXPECTED_EOF_IN_ARRAY = 0x18, + CBOR_UNEXPECTED_EOF_IN_MAP = 0x19, + CBOR_INVALID_MAP_KEY = 0x1a, + CBOR_STACK_LIMIT_EXCEEDED = 0x1b, + CBOR_TRAILING_JUNK = 0x1c, + CBOR_MAP_START_EXPECTED = 0x1d, + CBOR_MAP_STOP_EXPECTED = 0x1e, + CBOR_ENVELOPE_SIZE_LIMIT_EXCEEDED = 0x1f, +}; + +// A status value with position that can be copied. The default status +// is OK. Usually, error status values should come with a valid position. +struct Status { + static constexpr size_t npos() { return std::numeric_limits::max(); } + + bool ok() const { return error == Error::OK; } + + Error error = Error::OK; + size_t pos = npos(); + Status(Error error, size_t pos) : error(error), pos(pos) {} + Status() = default; + + // Returns a 7 bit US-ASCII string, either "OK" or an error message + // that includes the position. + std::string ToASCIIString() const; + + private: + std::string ToASCIIString(const char* msg) const; +}; + +// Handler interface for parser events emitted by a streaming parser. +// See cbor::NewCBOREncoder, cbor::ParseCBOR, json::NewJSONEncoder, +// json::ParseJSON. +class StreamingParserHandler { + public: + virtual ~StreamingParserHandler() = default; + virtual void HandleMapBegin() = 0; + virtual void HandleMapEnd() = 0; + virtual void HandleArrayBegin() = 0; + virtual void HandleArrayEnd() = 0; + virtual void HandleString8(span chars) = 0; + virtual void HandleString16(span chars) = 0; + virtual void HandleBinary(span bytes) = 0; + virtual void HandleDouble(double value) = 0; + virtual void HandleInt32(int32_t value) = 0; + virtual void HandleBool(bool value) = 0; + virtual void HandleNull() = 0; + + // The parser may send one error even after other events have already + // been received. Client code is reponsible to then discard the + // already processed events. + // |error| must be an eror, as in, |error.is_ok()| can't be true. + virtual void HandleError(Status error) = 0; +}; + +namespace cbor { +// The binary encoding for the inspector protocol follows the CBOR specification +// (RFC 7049). Additional constraints: +// - Only indefinite length maps and arrays are supported. +// - Maps and arrays are wrapped with an envelope, that is, a +// CBOR tag with value 24 followed by a byte string specifying +// the byte length of the enclosed map / array. The byte string +// must use a 32 bit wide length. +// - At the top level, a message must be an indefinite length map +// wrapped by an envelope. +// - Maximal size for messages is 2^32 (4 GB). +// - For scalars, we support only the int32_t range, encoded as +// UNSIGNED/NEGATIVE (major types 0 / 1). +// - UTF16 strings, including with unbalanced surrogate pairs, are encoded +// as CBOR BYTE_STRING (major type 2). For such strings, the number of +// bytes encoded must be even. +// - UTF8 strings (major type 3) are supported. +// - 7 bit US-ASCII strings must always be encoded as UTF8 strings, never +// as UTF16 strings. +// - Arbitrary byte arrays, in the inspector protocol called 'binary', +// are encoded as BYTE_STRING (major type 2), prefixed with a byte +// indicating base64 when rendered as JSON. + +// ============================================================================= +// Detecting CBOR content +// ============================================================================= + +// The first byte for an envelope, which we use for wrapping dictionaries +// and arrays; and the byte that indicates a byte string with 32 bit length. +// These two bytes start an envelope, and thereby also any CBOR message +// produced or consumed by this protocol. See also |EnvelopeEncoder| below. +uint8_t InitialByteForEnvelope(); +uint8_t InitialByteFor32BitLengthByteString(); + +// Checks whether |msg| is a cbor message. +bool IsCBORMessage(span msg); + +// ============================================================================= +// Encoding individual CBOR items +// ============================================================================= + +// Some constants for CBOR tokens that only take a single byte on the wire. +uint8_t EncodeTrue(); +uint8_t EncodeFalse(); +uint8_t EncodeNull(); +uint8_t EncodeIndefiniteLengthArrayStart(); +uint8_t EncodeIndefiniteLengthMapStart(); +uint8_t EncodeStop(); + +// Encodes |value| as |UNSIGNED| (major type 0) iff >= 0, or |NEGATIVE| +// (major type 1) iff < 0. +void EncodeInt32(int32_t value, std::vector* out); +void EncodeInt32(int32_t value, std::string* out); + +// Encodes a UTF16 string as a BYTE_STRING (major type 2). Each utf16 +// character in |in| is emitted with most significant byte first, +// appending to |out|. +void EncodeString16(span in, std::vector* out); +void EncodeString16(span in, std::string* out); + +// Encodes a UTF8 string |in| as STRING (major type 3). +void EncodeString8(span in, std::vector* out); +void EncodeString8(span in, std::string* out); + +// Encodes the given |latin1| string as STRING8. +// If any non-ASCII character is present, it will be represented +// as a 2 byte UTF8 sequence. +void EncodeFromLatin1(span latin1, std::vector* out); +void EncodeFromLatin1(span latin1, std::string* out); + +// Encodes the given |utf16| string as STRING8 if it's entirely US-ASCII. +// Otherwise, encodes as STRING16. +void EncodeFromUTF16(span utf16, std::vector* out); +void EncodeFromUTF16(span utf16, std::string* out); + +// Encodes arbitrary binary data in |in| as a BYTE_STRING (major type 2) with +// definitive length, prefixed with tag 22 indicating expected conversion to +// base64 (see RFC 7049, Table 3 and Section 2.4.4.2). +void EncodeBinary(span in, std::vector* out); +void EncodeBinary(span in, std::string* out); + +// Encodes / decodes a double as Major type 7 (SIMPLE_VALUE), +// with additional info = 27, followed by 8 bytes in big endian. +void EncodeDouble(double value, std::vector* out); +void EncodeDouble(double value, std::string* out); + +// ============================================================================= +// cbor::EnvelopeEncoder - for wrapping submessages +// ============================================================================= + +// An envelope indicates the byte length of a wrapped item. +// We use this for maps and array, which allows the decoder +// to skip such (nested) values whole sale. +// It's implemented as a CBOR tag (major type 6) with additional +// info = 24, followed by a byte string with a 32 bit length value; +// so the maximal structure that we can wrap is 2^32 bits long. +// See also: https://tools.ietf.org/html/rfc7049#section-2.4.4.1 +class EnvelopeEncoder { + public: + // Emits the envelope start bytes and records the position for the + // byte size in |byte_size_pos_|. Also emits empty bytes for the + // byte sisze so that encoding can continue. + void EncodeStart(std::vector* out); + void EncodeStart(std::string* out); + // This records the current size in |out| at position byte_size_pos_. + // Returns true iff successful. + bool EncodeStop(std::vector* out); + bool EncodeStop(std::string* out); + + private: + size_t byte_size_pos_ = 0; +}; + +// ============================================================================= +// cbor::NewCBOREncoder - for encoding from a streaming parser +// ============================================================================= + +// This can be used to convert to CBOR, by passing the return value to a parser +// that drives it. The handler will encode into |out|, and iff an error occurs +// it will set |status| to an error and clear |out|. Otherwise, |status.ok()| +// will be |true|. +std::unique_ptr NewCBOREncoder( + std::vector* out, + Status* status); +std::unique_ptr NewCBOREncoder(std::string* out, + Status* status); + +// ============================================================================= +// cbor::CBORTokenizer - for parsing individual CBOR items +// ============================================================================= + +// Tags for the tokens within a CBOR message that CBORTokenizer understands. +// Note that this is not the same terminology as the CBOR spec (RFC 7049), +// but rather, our adaptation. For instance, we lump unsigned and signed +// major type into INT32 here (and disallow values outside the int32_t range). +enum class CBORTokenTag { + // Encountered an error in the structure of the message. Consult + // status() for details. + ERROR_VALUE, + // Booleans and NULL. + TRUE_VALUE, + FALSE_VALUE, + NULL_VALUE, + // An int32_t (signed 32 bit integer). + INT32, + // A double (64 bit floating point). + DOUBLE, + // A UTF8 string. + STRING8, + // A UTF16 string. + STRING16, + // A binary string. + BINARY, + // Starts an indefinite length map; after the map start we expect + // alternating keys and values, followed by STOP. + MAP_START, + // Starts an indefinite length array; after the array start we + // expect values, followed by STOP. + ARRAY_START, + // Ends a map or an array. + STOP, + // An envelope indicator, wrapping a map or array. + // Internally this carries the byte length of the wrapped + // map or array. While CBORTokenizer::Next() will read / skip the entire + // envelope, CBORTokenizer::EnterEnvelope() reads the tokens + // inside of it. + ENVELOPE, + // We've reached the end there is nothing else to read. + DONE, +}; + +// The major types from RFC 7049 Section 2.1. +enum class MajorType { + UNSIGNED = 0, + NEGATIVE = 1, + BYTE_STRING = 2, + STRING = 3, + ARRAY = 4, + MAP = 5, + TAG = 6, + SIMPLE_VALUE = 7 +}; + +// CBORTokenizer segments a CBOR message, presenting the tokens therein as +// numbers, strings, etc. This is not a complete CBOR parser, but makes it much +// easier to implement one (e.g. ParseCBOR, above). It can also be used to parse +// messages partially. +class CBORTokenizer { + public: + explicit CBORTokenizer(span bytes); + ~CBORTokenizer(); + + // Identifies the current token that we're looking at, + // or ERROR_VALUE (in which ase ::Status() has details) + // or DONE (if we're past the last token). + CBORTokenTag TokenTag() const; + + // Advances to the next token. + void Next(); + // Can only be called if TokenTag() == CBORTokenTag::ENVELOPE. + // While Next() would skip past the entire envelope / what it's + // wrapping, EnterEnvelope positions the cursor inside of the envelope, + // letting the client explore the nested structure. + void EnterEnvelope(); + + // If TokenTag() is CBORTokenTag::ERROR_VALUE, then Status().error describes + // the error more precisely; otherwise it'll be set to Error::OK. + // In either case, Status().pos is the current position. + struct Status Status() const; + + // The following methods retrieve the token values. They can only + // be called if TokenTag() matches. + + // To be called only if ::TokenTag() == CBORTokenTag::INT32. + int32_t GetInt32() const; + + // To be called only if ::TokenTag() == CBORTokenTag::DOUBLE. + double GetDouble() const; + + // To be called only if ::TokenTag() == CBORTokenTag::STRING8. + span GetString8() const; + + // Wire representation for STRING16 is low byte first (little endian). + // To be called only if ::TokenTag() == CBORTokenTag::STRING16. + span GetString16WireRep() const; + + // To be called only if ::TokenTag() == CBORTokenTag::BINARY. + span GetBinary() const; + + // To be called only if ::TokenTag() == CBORTokenTag::ENVELOPE. + span GetEnvelopeContents() const; + + private: + void ReadNextToken(bool enter_envelope); + void SetToken(CBORTokenTag token, size_t token_byte_length); + void SetError(Error error); + + span bytes_; + CBORTokenTag token_tag_; + struct Status status_; + size_t token_byte_length_; + MajorType token_start_type_; + uint64_t token_start_internal_value_; +}; + +// ============================================================================= +// cbor::ParseCBOR - for receiving streaming parser events for CBOR messages +// ============================================================================= + +// Parses a CBOR encoded message from |bytes|, sending events to +// |out|. If an error occurs, sends |out->HandleError|, and parsing stops. +// The client is responsible for discarding the already received information in +// that case. +void ParseCBOR(span bytes, StreamingParserHandler* out); + +// ============================================================================= +// cbor::AppendString8EntryToMap - for limited in-place editing of messages +// ============================================================================= + +// Modifies the |cbor| message by appending a new key/value entry at the end +// of the map. Patches up the envelope size; Status.ok() iff successful. +// If not successful, |cbor| may be corrupted after this call. +Status AppendString8EntryToCBORMap(span string8_key, + span string8_value, + std::vector* cbor); +Status AppendString8EntryToCBORMap(span string8_key, + span string8_value, + std::string* cbor); + +namespace internals { // Exposed only for writing tests. +int8_t ReadTokenStart(span bytes, + cbor::MajorType* type, + uint64_t* value); + +void WriteTokenStart(cbor::MajorType type, + uint64_t value, + std::vector* encoded); +void WriteTokenStart(cbor::MajorType type, + uint64_t value, + std::string* encoded); +} // namespace internals +} // namespace cbor + +namespace json { +// Client code must provide an instance. Implementation should delegate +// to whatever is appropriate. +class Platform { + public: + virtual ~Platform() = default; + // Parses |str| into |result|. Returns false iff there are + // leftover characters or parsing errors. + virtual bool StrToD(const char* str, double* result) const = 0; + + // Prints |value| in a format suitable for JSON. + virtual std::unique_ptr DToStr(double value) const = 0; +}; + +// ============================================================================= +// json::NewJSONEncoder - for encoding streaming parser events as JSON +// ============================================================================= + +// Returns a handler object which will write ascii characters to |out|. +// |status->ok()| will be false iff the handler routine HandleError() is called. +// In that case, we'll stop emitting output. +// Except for calling the HandleError routine at any time, the client +// code must call the Handle* methods in an order in which they'd occur +// in valid JSON; otherwise we may crash (the code uses assert). +std::unique_ptr NewJSONEncoder( + const Platform* platform, + std::vector* out, + Status* status); +std::unique_ptr NewJSONEncoder(const Platform* platform, + std::string* out, + Status* status); + +// ============================================================================= +// json::ParseJSON - for receiving streaming parser events for JSON +// ============================================================================= + +void ParseJSON(const Platform& platform, + span chars, + StreamingParserHandler* handler); +void ParseJSON(const Platform& platform, + span chars, + StreamingParserHandler* handler); + +// ============================================================================= +// json::ConvertCBORToJSON, json::ConvertJSONToCBOR - for transcoding +// ============================================================================= +Status ConvertCBORToJSON(const Platform& platform, + span cbor, + std::string* json); +Status ConvertCBORToJSON(const Platform& platform, + span cbor, + std::vector* json); +Status ConvertJSONToCBOR(const Platform& platform, + span json, + std::vector* cbor); +Status ConvertJSONToCBOR(const Platform& platform, + span json, + std::vector* cbor); +Status ConvertJSONToCBOR(const Platform& platform, + span json, + std::string* cbor); +Status ConvertJSONToCBOR(const Platform& platform, + span json, + std::string* cbor); +} // namespace json + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} +#endif // !defined({{"_".join(config.protocol.namespace)}}_encoding_h) diff --git a/tools/inspector_protocol/pdl.py b/tools/inspector_protocol/pdl.py index 43111e944b4f5c..03d11b39d636df 100644 --- a/tools/inspector_protocol/pdl.py +++ b/tools/inspector_protocol/pdl.py @@ -74,20 +74,20 @@ def parse(data, file_name, map_binary_to_string=False): if len(trimLine) == 0: continue - match = re.compile('^(experimental )?(deprecated )?domain (.*)').match(line) + match = re.compile(r'^(experimental )?(deprecated )?domain (.*)').match(line) if match: domain = createItem({'domain' : match.group(3)}, match.group(1), match.group(2)) protocol['domains'].append(domain) continue - match = re.compile('^ depends on ([^\s]+)').match(line) + match = re.compile(r'^ depends on ([^\s]+)').match(line) if match: if 'dependencies' not in domain: domain['dependencies'] = [] domain['dependencies'].append(match.group(1)) continue - match = re.compile('^ (experimental )?(deprecated )?type (.*) extends (array of )?([^\s]+)').match(line) + match = re.compile(r'^ (experimental )?(deprecated )?type (.*) extends (array of )?([^\s]+)').match(line) if match: if 'types' not in domain: domain['types'] = [] @@ -96,7 +96,7 @@ def parse(data, file_name, map_binary_to_string=False): domain['types'].append(item) continue - match = re.compile('^ (experimental )?(deprecated )?(command|event) (.*)').match(line) + match = re.compile(r'^ (experimental )?(deprecated )?(command|event) (.*)').match(line) if match: list = [] if match.group(3) == 'command': @@ -114,7 +114,7 @@ def parse(data, file_name, map_binary_to_string=False): list.append(item) continue - match = re.compile('^ (experimental )?(deprecated )?(optional )?(array of )?([^\s]+) ([^\s]+)').match(line) + match = re.compile(r'^ (experimental )?(deprecated )?(optional )?(array of )?([^\s]+) ([^\s]+)').match(line) if match: param = createItem({}, match.group(1), match.group(2), match.group(6)) if match.group(3): @@ -125,36 +125,36 @@ def parse(data, file_name, map_binary_to_string=False): subitems.append(param) continue - match = re.compile('^ (parameters|returns|properties)').match(line) + match = re.compile(r'^ (parameters|returns|properties)').match(line) if match: subitems = item[match.group(1)] = [] continue - match = re.compile('^ enum').match(line) + match = re.compile(r'^ enum').match(line) if match: enumliterals = item['enum'] = [] continue - match = re.compile('^version').match(line) + match = re.compile(r'^version').match(line) if match: continue - match = re.compile('^ major (\d+)').match(line) + match = re.compile(r'^ major (\d+)').match(line) if match: protocol['version']['major'] = match.group(1) continue - match = re.compile('^ minor (\d+)').match(line) + match = re.compile(r'^ minor (\d+)').match(line) if match: protocol['version']['minor'] = match.group(1) continue - match = re.compile('^ redirect ([^\s]+)').match(line) + match = re.compile(r'^ redirect ([^\s]+)').match(line) if match: item['redirect'] = match.group(1) continue - match = re.compile('^ ( )?[^\s]+$').match(line) + match = re.compile(r'^ ( )?[^\s]+$').match(line) if match: # enum literal enumliterals.append(trimLine) diff --git a/tools/inspector_protocol/roll.py b/tools/inspector_protocol/roll.py new file mode 100644 index 00000000000000..abe636e270b7cf --- /dev/null +++ b/tools/inspector_protocol/roll.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python +# Copyright 2019 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from __future__ import print_function +import argparse +import sys +import os +import subprocess +import glob +import shutil + + +FILES_TO_SYNC = [ + 'README.md', + 'check_protocol_compatibility.py', + 'code_generator.py', + 'concatenate_protocols.py', + 'convert_protocol_to_json.py', + 'encoding/encoding.h', + 'encoding/encoding.cc', + 'encoding/encoding_test.cc', + 'inspector_protocol.gni', + 'inspector_protocol.gypi', + 'lib/*', + 'pdl.py', + 'templates/*', +] + + +def RunCmd(cmd): + p = subprocess.Popen(cmd, stdout=subprocess.PIPE) + (stdoutdata, stderrdata) = p.communicate() + if p.returncode != 0: + raise Exception('%s: exit status %d', str(cmd), p.returncode) + return stdoutdata + + +def CheckRepoIsClean(path, suffix): + os.chdir(path) # As a side effect this also checks for existence of the dir. + # If path isn't a git repo, this will throw and exception. + # And if it is a git repo and 'git status' has anything interesting to say, + # then it's not clean (uncommitted files etc.) + if len(RunCmd(['git', 'status', '--porcelain'])) != 0: + raise Exception('%s is not a clean git repo (run git status)' % path) + if not path.endswith(suffix): + raise Exception('%s does not end with /%s' % (path, suffix)) + + +def CheckRepoIsNotAtMasterBranch(path): + os.chdir(path) + stdout = RunCmd(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() + if stdout == 'master': + raise Exception('%s is at master branch - refusing to copy there.' % path) + + +def CheckRepoIsV8Checkout(path): + os.chdir(path) + if (RunCmd(['git', 'config', '--get', 'remote.origin.url']).strip() != + 'https://chromium.googlesource.com/v8/v8.git'): + raise Exception('%s is not a proper V8 checkout.' % path) + + +def CheckRepoIsInspectorProtocolCheckout(path): + os.chdir(path) + if (RunCmd(['git', 'config', '--get', 'remote.origin.url']).strip() != + 'https://chromium.googlesource.com/deps/inspector_protocol.git'): + raise Exception('%s is not a proper inspector_protocol checkout.' % path) + + +def FindFilesToSyncIn(path): + files = [] + for f in FILES_TO_SYNC: + files += glob.glob(os.path.join(path, f)) + files = [os.path.relpath(f, path) for f in files] + return files + + +def FilesAreEqual(path1, path2): + # We check for permissions (useful for executable scripts) and contents. + return (os.stat(path1).st_mode == os.stat(path2).st_mode and + open(path1).read() == open(path2).read()) + + +def GetHeadRevision(path): + os.chdir(path) + return RunCmd(['git', 'rev-parse', 'HEAD']) + + +def main(argv): + parser = argparse.ArgumentParser(description=( + "Rolls the inspector_protocol project (upstream) into V8's " + "third_party (downstream).")) + parser.add_argument("--ip_src_upstream", + help="The inspector_protocol (upstream) tree.", + default="~/ip/src") + parser.add_argument("--v8_src_downstream", + help="The V8 src tree.", + default="~/v8/v8") + parser.add_argument('--force', dest='force', action='store_true', + help=("Whether to carry out the modifications " + "in the destination tree.")) + parser.set_defaults(force=False) + + args = parser.parse_args(argv) + upstream = os.path.normpath(os.path.expanduser(args.ip_src_upstream)) + downstream = os.path.normpath(os.path.expanduser( + args.v8_src_downstream)) + CheckRepoIsClean(upstream, '/src') + CheckRepoIsClean(downstream, '/v8') + CheckRepoIsInspectorProtocolCheckout(upstream) + CheckRepoIsV8Checkout(downstream) + # Check that the destination Git repo isn't at the master branch - it's + # generally a bad idea to check into the master branch, so we catch this + # common pilot error here early. + CheckRepoIsNotAtMasterBranch(downstream) + src_dir = upstream + dest_dir = os.path.join(downstream, 'third_party/inspector_protocol') + print('Rolling %s into %s ...' % (src_dir, dest_dir)) + src_files = set(FindFilesToSyncIn(src_dir)) + dest_files = set(FindFilesToSyncIn(dest_dir)) + to_add = [f for f in src_files if f not in dest_files] + to_delete = [f for f in dest_files if f not in src_files] + to_copy = [f for f in src_files + if (f in dest_files and not FilesAreEqual( + os.path.join(src_dir, f), os.path.join(dest_dir, f)))] + print('To add: %s' % to_add) + print('To delete: %s' % to_delete) + print('To copy: %s' % to_copy) + if not to_add and not to_delete and not to_copy: + print('Nothing to do. You\'re good.') + sys.exit(0) + if not args.force: + print('Rerun with --force if you wish the modifications to be done.') + sys.exit(1) + print('You said --force ... as you wish, modifying the destination.') + for f in to_add + to_copy: + contents = open(os.path.join(src_dir, f)).read() + contents = contents.replace( + 'INSPECTOR_PROTOCOL_ENCODING_ENCODING_H_', + 'V8_INSPECTOR_PROTOCOL_ENCODING_ENCODING_H_') + contents = contents.replace( + 'namespace inspector_protocol_encoding', + 'namespace v8_inspector_protocol_encoding') + open(os.path.join(dest_dir, f), 'w').write(contents) + shutil.copymode(os.path.join(src_dir, f), os.path.join(dest_dir, f)) + for f in to_delete: + os.unlink(os.path.join(dest_dir, f)) + head_revision = GetHeadRevision(upstream) + lines = open(os.path.join(dest_dir, 'README.v8')).readlines() + f = open(os.path.join(dest_dir, 'README.v8'), 'w') + for line in lines: + if line.startswith('Revision: '): + f.write('Revision: %s' % head_revision) + else: + f.write(line) + f.close() + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/tools/inspector_protocol/templates/TypeBuilder_cpp.template b/tools/inspector_protocol/templates/TypeBuilder_cpp.template index 4ef60a6ea2cdef..982e2c61b8e916 100644 --- a/tools/inspector_protocol/templates/TypeBuilder_cpp.template +++ b/tools/inspector_protocol/templates/TypeBuilder_cpp.template @@ -203,12 +203,12 @@ void Frontend::flush() m_frontendChannel->flushProtocolNotifications(); } -void Frontend::sendRawNotification(String notification) +void Frontend::sendRawJSONNotification(String notification) { m_frontendChannel->sendProtocolNotification(InternalRawNotification::fromJSON(std::move(notification))); } -void Frontend::sendRawNotification(std::vector notification) +void Frontend::sendRawCBORNotification(std::vector notification) { m_frontendChannel->sendProtocolNotification(InternalRawNotification::fromBinary(std::move(notification))); } diff --git a/tools/inspector_protocol/templates/TypeBuilder_h.template b/tools/inspector_protocol/templates/TypeBuilder_h.template index c670d65c46f20d..9d86d7a4ac0a5c 100644 --- a/tools/inspector_protocol/templates/TypeBuilder_h.template +++ b/tools/inspector_protocol/templates/TypeBuilder_h.template @@ -269,8 +269,8 @@ public: {% endfor %} void flush(); - void sendRawNotification(String); - void sendRawNotification(std::vector); + void sendRawJSONNotification(String); + void sendRawCBORNotification(std::vector); private: FrontendChannel* m_frontendChannel; }; diff --git a/tools/js2c.py b/tools/js2c.py index 9c5130a91630c3..c3ac53f14b7391 100755 --- a/tools/js2c.py +++ b/tools/js2c.py @@ -177,6 +177,7 @@ def ReadMacros(macro_files): TEMPLATE = """ +#include "env-inl.h" #include "node_native_module.h" #include "node_internals.h" @@ -199,6 +200,12 @@ def ReadMacros(macro_files): }} // namespace node """ +ONE_BYTE_STRING = """ +static const uint8_t {0}[] = {{ +{1} +}}; +""" + TWO_BYTE_STRING = """ static const uint16_t {0}[] = {{ {1} @@ -214,15 +221,25 @@ def ReadMacros(macro_files): is_verbose = False def GetDefinition(var, source, step=30): - encoded_source = bytearray(source, 'utf-16le') - code_points = [encoded_source[i] + (encoded_source[i+1] * 256) for i in range(0, len(encoded_source), 2)] + template = ONE_BYTE_STRING + code_points = [ord(c) for c in source] + if any(c > 127 for c in code_points): + template = TWO_BYTE_STRING + # Treat non-ASCII as UTF-8 and encode as UTF-16 Little Endian. + encoded_source = bytearray(source, 'utf-16le') + code_points = [ + encoded_source[i] + (encoded_source[i + 1] * 256) + for i in range(0, len(encoded_source), 2) + ] + # For easier debugging, align to the common 3 char for code-points. elements_s = ['%3s' % x for x in code_points] # Put no more then `step` code-points in a line. slices = [elements_s[i:i + step] for i in range(0, len(elements_s), step)] lines = [','.join(s) for s in slices] array_content = ',\n'.join(lines) - definition = TWO_BYTE_STRING.format(var, array_content) + definition = template.format(var, array_content) + return definition, len(code_points) diff --git a/tools/snapshot/node_mksnapshot.cc b/tools/snapshot/node_mksnapshot.cc index c273ba20b610e1..f52cccb705f53a 100644 --- a/tools/snapshot/node_mksnapshot.cc +++ b/tools/snapshot/node_mksnapshot.cc @@ -8,6 +8,7 @@ #include "libplatform/libplatform.h" #include "node_internals.h" #include "snapshot_builder.h" +#include "util-inl.h" #include "v8.h" #ifdef _WIN32 diff --git a/tools/snapshot/snapshot_builder.cc b/tools/snapshot/snapshot_builder.cc index 1faa5330ae118f..54a4a2e73da249 100644 --- a/tools/snapshot/snapshot_builder.cc +++ b/tools/snapshot/snapshot_builder.cc @@ -1,7 +1,6 @@ #include "snapshot_builder.h" #include #include -#include "env-inl.h" #include "node_internals.h" #include "node_main_instance.h" #include "node_v8_platform-inl.h" diff --git a/vcbuild.bat b/vcbuild.bat index 807910d749cb89..4c84473558614e 100644 --- a/vcbuild.bat +++ b/vcbuild.bat @@ -328,8 +328,9 @@ if "%target%" == "Clean" goto exit :after-build rd %config% if errorlevel 1 echo "Old build output exists at 'out\%config%'. Please remove." & exit /B -if EXIST out\%config% mklink /D %config% out\%config% -if errorlevel 1 exit /B +:: Use /J because /D (symlink) requires special permissions. +if EXIST out\%config% mklink /J %config% out\%config% +if errorlevel 1 echo "Could not create junction to 'out\%config%'." & exit /B :sign @rem Skip signing unless the `sign` option was specified.