From fb9281ee73a4cd5eaf378ab0f4cfbacbed5a74c3 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 5 Aug 2021 13:00:26 +0200 Subject: [PATCH 01/56] fix: implement fetch per spec Also ports node-fetch tests. --- .github/workflows/nodejs.yml | 23 +- README.md | 6 +- docs/api/Dispatcher.md | 2 - index.js | 11 +- lib/api/api-fetch/LICENSE | 21 + lib/api/api-fetch/body.js | 182 ++- lib/api/api-fetch/constants.js | 119 ++ lib/api/api-fetch/headers.js | 215 +++- lib/api/api-fetch/index.js | 1304 ++++++++++++++++--- lib/api/api-fetch/request.js | 652 ++++++++++ lib/api/api-fetch/response.js | 383 ++++-- lib/api/api-fetch/symbols.js | 10 +- lib/api/api-fetch/util.js | 100 ++ lib/api/index.js | 3 + lib/api/readable.js | 2 +- lib/core/errors.js | 9 + lib/core/request.js | 3 + lib/core/util.js | 4 + package.json | 16 +- test/client-fetch.js | 28 +- test/node-fetch/LICENSE | 22 + test/node-fetch/headers.js | 282 +++++ test/node-fetch/main.js | 1662 +++++++++++++++++++++++++ test/node-fetch/request.js | 259 ++++ test/node-fetch/response.js | 218 ++++ test/node-fetch/utils/chai-timeout.js | 15 + test/node-fetch/utils/dummy.txt | 1 + test/node-fetch/utils/read-stream.js | 9 + test/node-fetch/utils/server.js | 437 +++++++ test/node-fetch/utils/toWeb.js | 73 ++ 30 files changed, 5704 insertions(+), 367 deletions(-) create mode 100644 lib/api/api-fetch/LICENSE create mode 100644 lib/api/api-fetch/constants.js create mode 100644 lib/api/api-fetch/request.js create mode 100644 lib/api/api-fetch/util.js create mode 100644 test/node-fetch/LICENSE create mode 100644 test/node-fetch/headers.js create mode 100644 test/node-fetch/main.js create mode 100644 test/node-fetch/request.js create mode 100644 test/node-fetch/response.js create mode 100644 test/node-fetch/utils/chai-timeout.js create mode 100644 test/node-fetch/utils/dummy.txt create mode 100644 test/node-fetch/utils/read-stream.js create mode 100644 test/node-fetch/utils/server.js create mode 100644 test/node-fetch/utils/toWeb.js diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml index 7802f2f7058..0dd4f6f33e6 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/nodejs.yml @@ -20,7 +20,7 @@ jobs: - uses: actions/checkout@v2 - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 + uses: actions/setup-node@v2 with: node-version: ${{ matrix.node-version }} @@ -28,17 +28,26 @@ jobs: run: | npm install - - name: Unit test (no coverage) + - name: Test Tap run: | - npm test + npm run test:tap - - name: Unit test (coverage) + - name: Test Jest run: | - npm run coverage:ci + npm run test:jest + + # How to run only for Node 16+? + # - name: Test node-fetch + # run: | + # npm run test:node-fetch - - name: Test types + - name: Test Types run: | npm run test:typescript - - name: Coverage report + - name: Coverage + run: | + npm run coverage:ci + + - name: Coverage Report uses: codecov/codecov-action@v1 diff --git a/README.md b/README.md index 002b50dd9b9..727ffe419b4 100644 --- a/README.md +++ b/README.md @@ -157,7 +157,7 @@ See [Dispatcher.connect](docs/api/Dispatcher.md#dispatcherconnect) for more deta https://fetch.spec.whatwg.org/ -### `undici.fetch([url, options]): Promise` +### `undici.fetch(input[, init]): Promise` Implements [fetch](https://fetch.spec.whatwg.org/). @@ -167,8 +167,8 @@ This is [experimental](https://nodejs.org/api/documentation.html#documentation_s Arguments: -* **url** `string | URL | object` -* **options** `RequestInit` +* **input** `string | Request` +* **init** `RequestInit` Returns: `Promise` diff --git a/docs/api/Dispatcher.md b/docs/api/Dispatcher.md index 36a1c26796c..0ac5bb298b2 100644 --- a/docs/api/Dispatcher.md +++ b/docs/api/Dispatcher.md @@ -307,8 +307,6 @@ client.dispatch({ ### `Dispatcher.fetch(options)` -Implements [fetch](https://fetch.spec.whatwg.org/). - Only supported on Node 16+. This is [experimental](https://nodejs.org/api/documentation.html#documentation_stability_index) and is not yet fully compliant the Fetch Standard. We plan to ship breaking changes to this feature until it is out of experimental. diff --git a/index.js b/index.js index b4b90b30033..d2f7c1e9fc4 100644 --- a/index.js +++ b/index.js @@ -84,7 +84,16 @@ function makeDispatcher (fn) { module.exports.setGlobalDispatcher = setGlobalDispatcher module.exports.getGlobalDispatcher = getGlobalDispatcher -module.exports.fetch = makeDispatcher(api.fetch) +if (api.fetch) { + module.exports.fetch = async function fetch (resource, init) { + const dispatcher = getGlobalDispatcher() + return api.fetch.call(dispatcher, resource, init) + } + module.exports.Headers = api.fetch.Headers + module.exports.Response = api.fetch.Response + module.exports.Request = api.fetch.Request +} + module.exports.request = makeDispatcher(api.request) module.exports.stream = makeDispatcher(api.stream) module.exports.pipeline = makeDispatcher(api.pipeline) diff --git a/lib/api/api-fetch/LICENSE b/lib/api/api-fetch/LICENSE new file mode 100644 index 00000000000..294350045bb --- /dev/null +++ b/lib/api/api-fetch/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Ethan Arrowood + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/lib/api/api-fetch/body.js b/lib/api/api-fetch/body.js index a012ded0496..cddbb1b0d15 100644 --- a/lib/api/api-fetch/body.js +++ b/lib/api/api-fetch/body.js @@ -1,14 +1,22 @@ 'use strict' const util = require('../../core/util') -const { Readable } = require('stream') +const { kState } = require('./symbols') +const { Blob } = require('buffer') +const { NotSupportedError } = require('../../core/errors') +const assert = require('assert') -let TransformStream +let ReadableStream // https://fetch.spec.whatwg.org/#concept-bodyinit-extract function extractBody (body) { // TODO: FormBody + let stream = null + let source = null + let length = null + let type = null + if (body == null) { return [null, null] } else if (body instanceof URLSearchParams) { @@ -16,51 +24,155 @@ function extractBody (body) { // this is implemented in Node.js as apart of an URLSearchParams instance toString method // See: https://github.com/nodejs/node/blob/e46c680bf2b211bbd52cf959ca17ee98c7f657f5/lib/internal/url.js#L490 // and https://github.com/nodejs/node/blob/e46c680bf2b211bbd52cf959ca17ee98c7f657f5/lib/internal/url.js#L1100 - return [{ - source: body.toString() - }, 'application/x-www-form-urlencoded;charset=UTF-8'] - } else if (typeof body === 'string') { - return [{ - source: body - }, 'text/plain;charset=UTF-8'] + source = body.toString() + length = Buffer.byteLength(source) + type = 'application/x-www-form-urlencoded;charset=UTF-8' } else if (body instanceof ArrayBuffer || ArrayBuffer.isView(body)) { - return [{ - source: body - }, null] + if (body instanceof DataView) { + // TODO: Blob doesn't seem to work with DataView? + body = body.buffer + } + source = body + length = Buffer.byteLength(body) } else if (util.isBlob(body)) { - return [{ - source: body, - length: body.size - }, body.type || null] - } else if (util.isStream(body) || typeof body.pipeThrough === 'function') { + source = body + length = body.size + type = body.type + } else if (typeof body.pipeThrough === 'function') { if (util.isDisturbed(body)) { throw new TypeError('disturbed') } - let stream - if (util.isStream(body)) { - stream = Readable.toWeb(body) - } else { - if (body.locked) { - throw new TypeError('locked') + if (body.locked) { + throw new TypeError('locked') + } + + stream = body + } else { + source = String(body) + length = Buffer.byteLength(source) + type = 'text/plain;charset=UTF-8' + } + + if (!stream) { + assert(source != null) + if (!ReadableStream) { + ReadableStream = require('stream/web').ReadableStream + } + stream = new ReadableStream({ + async start (controller) { + controller.enqueue(source) + controller.close() + }, + async pull () { + }, + async cancel (reason) { } + }) + } + + return [{ stream, source, length }, type] +} - if (!TransformStream) { - TransformStream = require('stream/web').TransformStream +function cloneBody (src) { + if (!src) { + return + } + + if (util.isDisturbed(src)) { + throw new TypeError('disturbed') + } + + if (src.locked) { + throw new TypeError('locked') + } + + // https://fetch.spec.whatwg.org/#concept-body-clone + const [out1, out2] = src.stream.tee() + src.stream = out1 + return { + stream: out2, + length: src.length, + source: src.source + } +} + +function safelyExtractBody (body) { + if (util.isDisturbed(body)) { + throw new TypeError('disturbed') + } + + if (body && body.locked) { + throw new TypeError('locked') + } + + return extractBody(body) +} + +const methods = { + async blob () { + const chunks = [] + + if (this[kState].body) { + if (this.bodyUsed || this[kState].body.stream.locked) { + throw new TypeError('unusable') } - // https://streams.spec.whatwg.org/#readablestream-create-a-proxy - const identityTransform = new TransformStream() - body.pipeThrough(identityTransform) - stream = identityTransform + this[kState].bodyUsed = true + for await (const chunk of this[kState].body.stream) { + chunks.push(chunk) + } } - return [{ - stream - }, null] - } else { - throw Error('Cannot extract Body from input: ', body) + return new Blob(chunks, { type: this.headers.get('Content-Type') || '' }) + }, + + async arrayBuffer () { + const blob = await this.blob() + return await blob.arrayBuffer() + }, + + async text () { + const blob = await this.blob() + return await blob.text() + }, + + async json () { + return JSON.parse(await this.text()) + }, + + async formData () { + // TODO: Implement. + throw new NotSupportedError('formData') + } +} + +const properties = { + body: { + enumerable: true, + get () { + return this[kState].body ? this[kState].body.stream : null + } + }, + bodyUsed: { + enumerable: true, + get () { + return this[kState].body && ( + util.isDisturbed(this[kState].body.stream) || + !!this[kState].bodyUsed + ) + } } } -module.exports = { extractBody } +function mixinBody (prototype) { + Object.assign(prototype, methods) + Object.defineProperties(prototype, properties) +} + +module.exports = { + extractBody, + safelyExtractBody, + cloneBody, + mixinBody +} diff --git a/lib/api/api-fetch/constants.js b/lib/api/api-fetch/constants.js new file mode 100644 index 00000000000..1e750d87342 --- /dev/null +++ b/lib/api/api-fetch/constants.js @@ -0,0 +1,119 @@ +'use strict' + +const forbiddenHeaderNames = [ + 'accept-charset', + 'accept-encoding', + 'access-control-request-headers', + 'access-control-request-method', + 'connection', + 'content-length', + 'cookie', + 'cookie2', + 'date', + 'dnt', + 'expect', + 'host', + 'keep-alive', + 'origin', + 'referer', + 'te', + 'trailer', + 'transfer-encoding', + 'upgrade', + 'via' +] + +const corsSafeListedMethods = [ + 'GET', + 'HEAD', + 'POST' +] + +const nullBodyStatus = [ + 101, + 204, + 205, + 304 +] + +const redirectStatus = [ + 301, + 302, + 303, + 307, + 308 +] + +const referrerPolicy = [ + '', + 'no-referrer', + 'no-referrer-when-downgrade', + 'same-origin', + 'origin', + 'strict-origin', + 'origin-when-cross-origin', + 'strict-origin-when-cross-origin', + 'unsafe-url' +] + +const requestRedirect = [ + 'follow', + 'manual', + 'error' +] + +const safeMethods = [ + 'GET', + 'HEAD', + 'OPTIONS', + 'TRACE' +] + +const requestMode = [ + 'navigate', + 'same-origin', + 'no-cors', + 'cors' +] + +const requestCredentials = [ + 'omit', + 'same-origin', + 'include' +] + +const requestCache = [ + 'default', + 'no-store', + 'reload', + 'no-cache', + 'force-cache', + 'only-if-cached' +] + +const forbiddenResponseHeaderNames = [ + 'set-cookie', + 'set-cookie2' +] + +const requestBodyHeader = [ + 'content-encoding', + 'content-language', + 'content-location', + 'content-type' +] + +module.exports = { + forbiddenResponseHeaderNames, + requestBodyHeader, + referrerPolicy, + requestRedirect, + requestMode, + requestCredentials, + requestCache, + forbiddenHeaderNames, + redirectStatus, + corsSafeListedMethods, + nullBodyStatus, + safeMethods +} diff --git a/lib/api/api-fetch/headers.js b/lib/api/api-fetch/headers.js index 967df35260b..d020541c0f7 100644 --- a/lib/api/api-fetch/headers.js +++ b/lib/api/api-fetch/headers.js @@ -5,7 +5,17 @@ const { types } = require('util') const { validateHeaderName, validateHeaderValue } = require('http') const { kHeadersList } = require('../../core/symbols') -const { InvalidHTTPTokenError, HTTPInvalidHeaderValueError, InvalidArgumentError, InvalidThisError } = require('../../core/errors') +const { kGuard } = require('./symbols') +const { kEnumerableProperty } = require('../../core/util') +const { + InvalidHTTPTokenError, + HTTPInvalidHeaderValueError, + InvalidThisError +} = require('../../core/errors') +const { + forbiddenHeaderNames, + forbiddenResponseHeaderNames +} = require('./constants') function binarySearch (arr, val) { let low = 0 @@ -49,21 +59,21 @@ function isHeaders (object) { function fill (headers, object) { if (isHeaders(object)) { // Object is instance of Headers - headers[kHeadersList] = Array.splice(object[kHeadersList]) + headers[kHeadersList].push(...object[kHeadersList]) } else if (Array.isArray(object)) { // Support both 1D and 2D arrays of header entries if (Array.isArray(object[0])) { // Array of arrays for (let i = 0; i < object.length; i++) { if (object[i].length !== 2) { - throw new InvalidArgumentError(`The argument 'init' is not of length 2. Received ${object[i]}`) + throw new TypeError(`The argument 'init' is not of length 2. Received ${object[i]}`) } headers.append(object[i][0], object[i][1]) } } else if (typeof object[0] === 'string' || Buffer.isBuffer(object[0])) { // Flat array of strings or Buffers if (object.length % 2 !== 0) { - throw new InvalidArgumentError(`The argument 'init' is not even in length. Received ${object}`) + throw new TypeError(`The argument 'init' is not even in length. Received ${object}`) } for (let i = 0; i < object.length; i += 2) { headers.append( @@ -73,7 +83,7 @@ function fill (headers, object) { } } else { // All other array based entries - throw new InvalidArgumentError(`The argument 'init' is not a valid array entry. Received ${object}`) + throw new TypeError(`The argument 'init' is not a valid array entry. Received ${object}`) } } else if (!types.isBoxedPrimitive(object)) { // Object of key/value entries @@ -90,95 +100,66 @@ function validateArgumentLength (found, expected) { } } -class Headers { - constructor (init = {}) { - // validateObject allowArray = true - if (!Array.isArray(init) && typeof init !== 'object') { - throw new InvalidArgumentError('The argument \'init\' must be one of type Object or Array') - } - this[kHeadersList] = [] - fill(this, init) - } - +class HeadersList extends Array { append (...args) { - if (!isHeaders(this)) { - throw new InvalidThisError('Header') - } - validateArgumentLength(args.length, 2) const [name, value] = args const normalizedName = normalizeAndValidateHeaderName(name) const normalizedValue = normalizeAndValidateHeaderValue(name, value) - const index = binarySearch(this[kHeadersList], normalizedName) + const index = binarySearch(this, normalizedName) - if (this[kHeadersList][index] === normalizedName) { - this[kHeadersList][index + 1] += `, ${normalizedValue}` + if (this[index] === normalizedName) { + this[index + 1] += `, ${normalizedValue}` } else { - this[kHeadersList].splice(index, 0, normalizedName, normalizedValue) + this.splice(index, 0, normalizedName, normalizedValue) } } delete (...args) { - if (!isHeaders(this)) { - throw new InvalidThisError('Header') - } - validateArgumentLength(args.length, 1) const [name] = args const normalizedName = normalizeAndValidateHeaderName(name) - const index = binarySearch(this[kHeadersList], normalizedName) + const index = binarySearch(this, normalizedName) - if (this[kHeadersList][index] === normalizedName) { - this[kHeadersList].splice(index, 2) + if (this[index] === normalizedName) { + this.splice(index, 2) } } get (...args) { - if (!isHeaders(this)) { - throw new InvalidThisError('Header') - } - validateArgumentLength(args.length, 1) const [name] = args const normalizedName = normalizeAndValidateHeaderName(name) - const index = binarySearch(this[kHeadersList], normalizedName) + const index = binarySearch(this, normalizedName) - if (this[kHeadersList][index] === normalizedName) { - return this[kHeadersList][index + 1] + if (this[index] === normalizedName) { + return this[index + 1] } return null } has (...args) { - if (!isHeaders(this)) { - throw new InvalidThisError('Header') - } - validateArgumentLength(args.length, 1) const [name] = args const normalizedName = normalizeAndValidateHeaderName(name) - const index = binarySearch(this[kHeadersList], normalizedName) + const index = binarySearch(this, normalizedName) - return this[kHeadersList][index] === normalizedName + return this[index] === normalizedName } set (...args) { - if (!isHeaders(this)) { - throw new InvalidThisError('Header') - } - validateArgumentLength(args.length, 2) const [name, value] = args @@ -186,12 +167,126 @@ class Headers { const normalizedName = normalizeAndValidateHeaderName(name) const normalizedValue = normalizeAndValidateHeaderValue(name, value) - const index = binarySearch(this[kHeadersList], normalizedName) - if (this[kHeadersList][index] === normalizedName) { - this[kHeadersList][index + 1] = normalizedValue + const index = binarySearch(this, normalizedName) + if (this[index] === normalizedName) { + this[index + 1] = normalizedValue } else { - this[kHeadersList].splice(index, 0, normalizedName, normalizedValue) + this.splice(index, 0, normalizedName, normalizedValue) + } + } +} + +class Headers { + constructor (init = {}) { + // validateObject allowArray = true + if (!Array.isArray(init) && typeof init !== 'object') { + throw new TypeError('The argument \'init\' must be one of type Object or Array') + } + this[kHeadersList] = new HeadersList() + this[kGuard] = 'none' + fill(this, init) + } + + get [Symbol.toStringTag] () { + return this.constructor.name + } + + toString () { + return Object.prototype.toString.call(this) + } + + append (...args) { + if (!isHeaders(this)) { + throw new InvalidThisError('Header') + } + + const normalizedName = normalizeAndValidateHeaderName(args[0]) + + if (this[kGuard] === 'immutable') { + throw new TypeError('immutable') + } else if ( + this[kGuard] === 'request' && + forbiddenHeaderNames.includes(normalizedName) + ) { + return + } else if (this[kGuard] === 'request-no-cors') { + // TODO + } else if ( + this[kGuard] === 'response' && + forbiddenResponseHeaderNames.includes(normalizedName) + ) { + return + } + + return this[kHeadersList].append(...args) + } + + delete (...args) { + if (!isHeaders(this)) { + throw new InvalidThisError('Header') + } + + const normalizedName = normalizeAndValidateHeaderName(args[0]) + + if (this[kGuard] === 'immutable') { + throw new TypeError('immutable') + } else if ( + this[kGuard] === 'request' && + forbiddenHeaderNames.includes(normalizedName) + ) { + return + } else if (this[kGuard] === 'request-no-cors') { + // TODO + } else if ( + this[kGuard] === 'response' && + forbiddenResponseHeaderNames.includes(normalizedName) + ) { + return + } + + return this[kHeadersList].delete(...args) + } + + get (...args) { + if (!isHeaders(this)) { + throw new InvalidThisError('Header') } + + return this[kHeadersList].get(...args) + } + + has (...args) { + if (!isHeaders(this)) { + throw new InvalidThisError('Header') + } + + return this[kHeadersList].has(...args) + } + + set (...args) { + if (!isHeaders(this)) { + throw new InvalidThisError('Header') + } + + const normalizedName = normalizeAndValidateHeaderName(args[0]) + + if (this[kGuard] === 'immutable') { + throw new TypeError('immutable') + } else if ( + this[kGuard] === 'request' && + forbiddenHeaderNames.includes(normalizedName) + ) { + return + } else if (this[kGuard] === 'request-no-cors') { + // TODO + } else if ( + this[kGuard] === 'response' && + forbiddenResponseHeaderNames.includes(normalizedName) + ) { + return + } + + return this[kHeadersList].set(...args) } * keys () { @@ -233,8 +328,24 @@ class Headers { callback.call(thisArg, this[kHeadersList][index + 1], this[kHeadersList][index], this) } } + + [Symbol.for('nodejs.util.inspect.custom')] () { + return this[kHeadersList] + } } Headers.prototype[Symbol.iterator] = Headers.prototype.entries -module.exports = Headers +Object.defineProperties(Headers.prototype, { + append: kEnumerableProperty, + delete: kEnumerableProperty, + get: kEnumerableProperty, + has: kEnumerableProperty, + set: kEnumerableProperty, + keys: kEnumerableProperty, + values: kEnumerableProperty, + entries: kEnumerableProperty, + forEach: kEnumerableProperty +}) + +module.exports = { fill, Headers, HeadersList } diff --git a/lib/api/api-fetch/index.js b/lib/api/api-fetch/index.js index 77426ef957b..9f9e82bd367 100644 --- a/lib/api/api-fetch/index.js +++ b/lib/api/api-fetch/index.js @@ -2,250 +2,1232 @@ 'use strict' -const Headers = require('./headers') -const { kHeadersList } = require('../../core/symbols') -const { METHODS } = require('http') -const Response = require('./response') +const { Response, makeNetworkError, filterResponse } = require('./response') +const { Headers } = require('./headers') +const { Request, makeRequest } = require('./request') +const { badPort } = require('./util') +const { kState, kHeaders, kGuard } = require('./symbols') +const { AbortError } = require('../../core/errors') +const assert = require('assert') +const { safelyExtractBody } = require('./body') +const { STATUS_CODES } = require('http') const { - InvalidArgumentError, - NotSupportedError, - RequestAbortedError -} = require('../../core/errors') -const { addSignal, removeSignal } = require('../abort-signal') -const { extractBody } = require('./body') + redirectStatus, + nullBodyStatus, + safeMethods, + requestBodyHeader +} = require('./constants') +const { kHeadersList } = require('../../core/symbols') let ReadableStream -class FetchHandler { - constructor (opts, callback) { - if (!opts || typeof opts !== 'object') { - throw new InvalidArgumentError('invalid opts') - } +// https://fetch.spec.whatwg.org/#fetch-method +async function fetch (resource, init) { + const context = { + dispatcher: this, + abort: null, + terminated: false, + terminate ({ aborted } = {}) { + if (this.terminated) { + return + } + + this.terminated = { aborted } - const { signal, method, opaque } = opts + assert(!this.abort || aborted) - if (typeof callback !== 'function') { - throw new InvalidArgumentError('invalid callback') + if (this.abort) { + this.abort() + this.abort = null + } } + } + + // 1. Let p be a new promise. + const p = createDeferredPromise() + + // 2. Let requestObject be the result of invoking the initial value of + // Request as constructor with input and init as arguments. If this throws + // an exception, reject p with it and return p. + const requestObject = new Request(resource, init) + + // 3. Let request be requestObject’s request. + const request = requestObject[kState] + + // 4. If requestObject’s signal’s aborted flag is set, then: + if (requestObject.signal.aborted) { + // 1. Abort fetch with p, request, and null. + abortFetch(p, request, null) + + // 2. Return p. + return p.promise + } + + // 5. Let globalObject be request’s client’s global object. + // TODO + + // 6. If globalObject is a ServiceWorkerGlobalScope object, then set + // request’s service-workers mode to "none". + // TODO + + // 7. Let responseObject be null. + let responseObject = null + + // 8. Let relevantRealm be this’s relevant Realm. + // TODO + + // 9. Let locallyAborted be false. + let locallyAborted = false - if (signal && typeof signal.on !== 'function' && typeof signal.addEventListener !== 'function') { - throw new InvalidArgumentError('signal must be an EventEmitter or EventTarget') + // 10. Add the following abort steps to requestObject’s signal: + requestObject.signal.addEventListener('abort', () => { + // 1. Set locallyAborted to true. + locallyAborted = true + + // 2. Abort fetch with p, request, and responseObject. + abortFetch(p, request, responseObject) + + // 3. Terminate the ongoing fetch with the aborted flag set. + context.terminate({ aborted: true }) + }, { once: true }) + + // 11. Let handleFetchDone given response response be to finalize and + // report timing with response, globalObject, and "fetch". + const handleFetchDone = (response) => finalizeAndReportTiming(response, 'fetch') + + // 12. Fetch request with processResponseDone set to handleFetchDone, + // and processResponse given response being these substeps: + const processResponse = (response) => { + // 1. If locallyAborted is true, terminate these substeps. + if (locallyAborted) { + return } - if (method === 'CONNECT') { - throw new InvalidArgumentError('invalid method') + // 2. If response’s aborted flag is set, then abort fetch with p, + // request, and responseObject, and terminate these substeps. + if (response.aborted) { + abortFetch(p, request, responseObject) + return } - this.opaque = opaque || null - this.callback = callback - this.controller = null + // 3. If response is a network error, then reject p with a TypeError + // and terminate these substeps. + if (response.status === 0) { + const error = new TypeError('fetch failed') + error.cause = response.error + p.reject(error) + return + } - this.abort = null - this.context = null - this.redirect = opts.redirect || 'follow' - this.url = new URL(opts.path, opts.origin) + // 4. Set responseObject to the result of creating a Response object, + // given response, "immutable", and relevantRealm. + // TODO: relevantRealm + responseObject = new Response() + responseObject[kState] = response + responseObject[kHeaders][kHeadersList] = response.headersList + responseObject[kHeaders][kGuard] = 'immutable' - addSignal(this, signal) + // 5. Resolve p with responseObject. + p.resolve(responseObject) } - onConnect (abort, context) { - if (!this.callback) { - throw new RequestAbortedError() + fetching.call(context, { + request, + processResponseDone: handleFetchDone, + processResponse + }) + + // 13. Return p. + return p.promise +} + +function finalizeAndReportTiming (response, initiatorType = 'other') { + // TODO +} + +// https://fetch.spec.whatwg.org/#abort-fetch +function abortFetch (p, request, responseObject) { + // 1. Let error be an "AbortError" DOMException. + const error = new AbortError() + + // 2. Reject promise with error. + p.reject(error) + + // 3. If request’s body is not null and is readable, then cancel request’s + // body with error. + if (request.body != null) { + // NOTE: We don't have any way to check if body is readable. However, + // cancel will be noop if state is "closed" and reject if state is + // "error", which means that calling cancel() and catch errors + // has the same effect as checking for "readable". + + try { + // TODO: What if locked? + request.body.stream.cancel(error) + } catch { + // Do nothing... } + } - this.abort = abort - this.context = context + // 4. If responseObject is null, then return. + if (responseObject == null) { + return } - onHeaders (statusCode, headers, resume) { - const { callback, abort, context } = this + // 5. Let response be responseObject’s response. + const response = responseObject[kState] - if (statusCode < 200) { - return + // 6. If response’s body is not null and is readable, then error response’s + // body with error. + if (response.body != null && context.controller) { + context.controller.error(error) + } +} + +// https://fetch.spec.whatwg.org/#fetching +function fetching ({ + request, + processResponse, + processResponseDone +}) { + // 1. Let taskDestination be null. + // TODO + + // 2. Let crossOriginIsolatedCapability be false. + // TODO + + // 3. If request’s client is non-null, then: + // TODO + + // 4. If useParallelQueue is true, then set taskDestination to the result of + // starting a new parallel queue. + + // 5. Let timingInfo be a new fetch timing info whose start time and + // post-redirect start time are the coarsened shared current time given + // crossOriginIsolatedCapability. + + // 6. Let fetchParams be a new fetch params whose request is request, timing + // info is timingInfo, process request body is processRequestBody, + // process request end-of-body is processRequestEndOfBody, process response + // is processResponse, process response end-of-body is + // processResponseEndOfBody, process response done is processResponseDone, + // task destination is taskDestination, and cross-origin isolated capability + // is crossOriginIsolatedCapability. + const fetchParams = { + request, + processRequestBody: null, + processRequestEndOfBody: null, + processResponse, + processResponseEndOfBody: null, + processResponseDone + } + + // 7. If request’s body is a byte sequence, then set request’s body to the + // first return value of safely extracting request’s body. + // TODO + + // 8. If request’s window is "client", then set request’s window to request’s + // client, if request’s client’s global object is a Window object; otherwise + // "no-window". + // TODO + + // 9. If request’s origin is "client", then set request’s origin to request’s + // client’s origin. + if (request.origin === 'client') { + // TODO: What is correct here? + request.origin = request.currentURL.origin + } + + // 10. If request’s policy container is "client", then: + if (request.policyContainer === 'client') { + // TODO + } + + // 11. If request’s header list does not contain `Accept`, then: + if (!request.headersList.has('accept')) { + // 1. Let value be `*/*`. + const value = '*/*' + + // 2. A user agent should set value to the first matching statement, if + // any, switching on request’s destination: + // "document" + // "frame" + // "iframe" + // `text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8` + // "image" + // `image/png,image/svg+xml,image/*;q=0.8,*/*;q=0.5` + // "style" + // `text/css,*/*;q=0.1` + // TODO + + // 3. Append `Accept`/value to request’s header list. + request.headersList.append('accept', value) + + // 12. If request’s header list does not contain `Accept-Language`, then + // user agents should append `Accept-Language`/an appropriate value to + // request’s header list. + if (!request.headersList.has('accept-language')) { + request.headersList.append('accept-language', '*') } + } - headers = new Headers(headers) + // 13. If request’s priority is null, then use request’s initiator and + // destination appropriately in setting request’s priority to a + // user-agent-defined object. + // TODO - let response - if (headers.has('location')) { - if (this.redirect === 'manual') { - response = new Response({ - type: 'opaqueredirect', - url: this.url - }) - } else { - response = new Response({ - type: 'error', - url: this.url - }) + // 14. If request is a subresource request, then: + // TODO + + // 15. Run main fetch given fetchParams. + mainFetch.call(this, fetchParams) +} + +// https://fetch.spec.whatwg.org/#concept-main-fetch +async function mainFetch (fetchParams, recursive = false) { + // 1. Let request be fetchParams’s request. + const request = fetchParams.request + + // 2. Let response be null. + let response = null + + // 3. If request’s local-URLs-only flag is set and request’s current URL is + // not local, then set response to a network error. + // TODO + + // 4. Run report Content Security Policy violations for request. + // TODO + + // 5. Upgrade request to a potentially trustworthy URL, if appropriate. + // TODO + + // 6. If should request be blocked due to a bad port, should fetching request + // be blocked as mixed content, or should request be blocked by Content + // Security Policy returns blocked, then set response to a network error. + if (badPort(request) === 'blocked') { + return makeNetworkError('bad port') + } + // TODO: should fetching request be blocked as mixed content? + // TODO: should request be blocked by Content Security Policy? + + // 7. If request’s referrer policy is the empty string, then set request’s + // referrer policy to request’s policy container’s referrer policy. + if (!request.referrerPolicy) { + // TODO + } + + // 8. If request’s referrer is not "no-referrer", then set request’s + // referrer to the result of invoking determine request’s referrer. + if (request.referrer !== 'no-referrer') { + // TODO + } + + // 9. Set request’s current URL’s scheme to "https" if all of the following + // conditions are true: + // - request’s current URL’s scheme is "http" + // - request’s current URL’s host is a domain + // - Matching request’s current URL’s host per Known HSTS Host Domain Name + // Matching results in either a superdomain match with an asserted + // includeSubDomains directive or a congruent match (with or without an + // asserted includeSubDomains directive). [HSTS] + // TODO + + // 10. If recursive is false, then run the remaining steps in parallel. + // TODO + + // 11. If response is null, then set response to the result of running + // the steps corresponding to the first matching statement: + // TODO + response = await (async () => { + // - request’s current URL’s origin is same origin with request’s origin, + // and request’s response tainting is "basic" + // - request’s current URL’s scheme is "data" + // - request’s mode is "navigate" or "websocket" + // 1. Set request’s response tainting to "basic". + // 2. Return the result of running scheme fetch given fetchParams. + // NOTE: This branch is not possible since we cannot set request mode + // to "navigate". + + // request’s mode is "same-origin" + if (request.mode === 'same-origin') { + // 1. Return a network error. + return makeNetworkError('request mode cannot be "same-origin"') + } + + // request’s mode is "no-cors" + if (request.mode === 'no-cors') { + // 1. If request’s redirect mode is not "follow", then return a network + // error. + if (request.redirect !== 'follow') { + return makeNetworkError('redirect cmode cannot be "follow" for "no-cors" request') } + + // 2. Set request’s response tainting to "opaque". + request.responseTainting = 'opaque' + + // 3. Let noCorsResponse be the result of running scheme fetch given + // fetchParams. + // TODO + + // 4. If noCorsResponse is a filtered response or the CORB check with + // request and noCorsResponse returns allowed, then return noCorsResponse. + // TODO + + // 5. Return a new response whose status is noCorsResponse’s status. + // TODO + } + + // request’s current URL’s scheme is not an HTTP(S) scheme + if (!/^https?:/.test(request.currentURL.protocol)) { + // Return a network error. + return makeNetworkError('URL scheme must be a HTTP(S) scheme') + } + + // - request’s use-CORS-preflight flag is set + // - request’s unsafe-request flag is set and either request’s method is + // not a CORS-safelisted method or CORS-unsafe request-header names with + // request’s header list is not empty + // 1. Set request’s response tainting to "cors". + // 2. Let corsWithPreflightResponse be the result of running HTTP fetch + // given fetchParams and true. + // 3. If corsWithPreflightResponse is a network error, then clear cache + // entries using request. + // 4. Return corsWithPreflightResponse. + // TODO + + // Otherwise + // 1. Set request’s response tainting to "cors". + request.responseTainting = 'cors' + + // 2. Return the result of running HTTP fetch given fetchParams. + return await httpFetch.call(this, fetchParams).catch((err) => makeNetworkError(err)) + })() + + // 12. If recursive is true, then return response. + if (recursive) { + return response + } + + // 13. If response is not a network error and response is not a filtered + // response, then: + if (response.status !== 0 && !response.internalResponse) { + // If request’s response tainting is "cors", then: + if (request.responseTainting === 'cors') { + // 1. Let headerNames be the result of extracting header list values + // given `Access-Control-Expose-Headers` and response’s header list. + // TODO + // 2. If request’s credentials mode is not "include" and headerNames + // contains `*`, then set response’s CORS-exposed header-name list to + // all unique header names in response’s header list. + // TODO + // 3. Otherwise, if headerNames is not null or failure, then set + // response’s CORS-exposed header-name list to headerNames. + // TODO + } + + // Set response to the following filtered response with response as its + // internal response, depending on request’s response tainting: + if (request.responseTainting === 'basic') { + response = filterResponse(response, 'basic') + } else if (request.responseTainting === 'cors') { + response = filterResponse(response, 'cors') + } else if (request.responseTainting === 'opaque') { + response = filterResponse(response, 'opaque') } else { - const self = this - if (!ReadableStream) { - ReadableStream = require('stream/web').ReadableStream - } - response = new Response({ - type: 'default', - url: this.url, - body: new ReadableStream({ - async start (controller) { - self.controller = controller - }, - async pull () { - resume() - }, - async cancel (reason) { - let err - if (reason instanceof Error) { - err = reason - } else if (typeof reason === 'string') { - err = new Error(reason) - } else { - err = new RequestAbortedError() - } - abort(err) - } - }, { highWaterMark: 16384 }), - statusCode, - headers, - context - }) + assert(false) } + } - this.callback = null - callback(null, response) + // 14. Let internalResponse be response, if response is a network error, + // and response’s internal response otherwise. + let internalResponse = response.status === 0 + ? response + : response.internalResponse - return false + // 15. If internalResponse’s URL list is empty, then set it to a clone of + // request’s URL list. + if (internalResponse.urlList.length === 0) { + internalResponse.urlList.push(...request.urlList) } - onData (chunk) { - const { controller } = this + // 16. If request’s timing allow failed flag is unset, then set + // internalResponse’s timing allow passed flag. + // TODO - // Copy the Buffer to detach it from Buffer pool. - // TODO: Is this required? - chunk = new Uint8Array(chunk) + // 17. If response is not a network error and any of the following returns + // blocked + // - should internalResponse to request be blocked as mixed content + // - should internalResponse to request be blocked by Content Security Policy + // - should internalResponse to request be blocked due to its MIME type + // - should internalResponse to request be blocked due to nosniff + // TODO + + // 18. If response’s type is "opaque", internalResponse’s status is 206, + // internalResponse’s range-requested flag is set, and request’s header + // list does not contain `Range`, then set response and internalResponse + // to a network error. + if ( + response.type === 'opaque' && + internalResponse.status === 206 && + internalResponse.rangeRequested && + !request.headers.has('range') + ) { + response = internalResponse = makeNetworkError() + } + + // 19. If response is not a network error and either request’s method is + // `HEAD` or `CONNECT`, or internalResponse’s status is a null body status, + // set internalResponse’s body to null and disregard any enqueuing toward + // it (if any). + if ( + response.status !== 0 && + ( + request.method === 'HEAD' || + request.method === 'CONNECT' || + nullBodyStatus.includes(internalResponse.status) + ) + ) { + internalResponse.body = null + } - controller.enqueue(chunk) + // 20. If request’s integrity metadata is not the empty string, then: + if (request.integrity) { + // 1. Let processBodyError be this step: run fetch finale given fetchParams + // and a network error. + const processBodyError = (reason) => fetchFinale(fetchParams, makeNetworkError(reason)) - return controller.desiredSize > 0 + // 2. If request’s response tainting is "opaque", response is a network + // error, or response’s body is null, then run processBodyError and abort + // these steps. + if ( + request.responseTainting === 'opaque' && + response.status === 0 + ) { + processBodyError(response.error) + return + } + + // 3. Let processBody given bytes be these steps: + const processBody = (bytes) => { + // 1. If bytes do not match request’s integrity metadata, + // then run processBodyError and abort these steps. [SRI] + // TODO + + // 2. Set response’s body to the first return value of safely + // extracting bytes. + response.body = safelyExtractBody(bytes)[0] + + // 3. Run fetch finale given fetchParams and response. + fetchFinale(fetchParams, response) + } + + // 4. Fully read response’s body given processBody and processBodyError. + try { + processBody(await response.arrayBuffer()) + } catch (err) { + processBodyError(err) + } + } else { + // 21. Otherwise, run fetch finale given fetchParams and response. + fetchFinale(fetchParams, response) } +} - onComplete () { - const { controller } = this +// https://fetch.spec.whatwg.org/#finalize-response +function finalizeResponse (fetchParams, response) { + // 1. Set fetchParams’s request’s done flag. + fetchParams.request.done = true - removeSignal(this) + // 2, If fetchParams’s process response done is not null, then queue a fetch + // task to run fetchParams’s process response done given response, with + // fetchParams’s task destination. + if (fetchParams.processResponseDone) { + fetchParams.processResponseDone(response) + } +} - controller.close() +// https://fetch.spec.whatwg.org/#fetch-finale +function fetchFinale (fetchParams, response) { + // 1. If fetchParams’s process response is non-null, + // then queue a fetch task to run fetchParams’s process response + // given response, with fetchParams’s task destination. + if (fetchParams.processResponse != null) { + fetchParams.processResponse(response) } - onError (err) { - const { controller, callback } = this + // 2. If fetchParams’s process response end-of-body is non-null, then:. + // TODO + // 1. Let processBody given nullOrBytes be this step: run fetchParams’s + // process response end-of-body given response and nullOrBytes.on. + // TODO + // 2. Let processBodyError be this step: run fetchParams’s process + // response end-of-body given response and failure.on. + // TODO + // 3. If response’s body is null, then queue a fetch task to run + // processBody given null, with fetchParams’s task destination.on. + // TODO + // 4. Otherwise, fully read response’s body given processBody, + // processBodyError, and fetchParams’s task destination.on. + // TODO +} + +// https://fetch.spec.whatwg.org/#http-fetch +async function httpFetch (fetchParams) { + // 1. Let request be fetchParams’s request. + const request = fetchParams.request + + // 2. Let response be null. + let response = null + + // 3. Let actualResponse be null. + let actualResponse = null + + // 4. Let timingInfo be fetchParams’s timing info. + // TODO - removeSignal(this) + // 5. If request’s service-workers mode is "all", then: + // TODO - if (callback) { - this.callback = null - callback(err) + // 6. If response is null, then: + if (response === null) { + // 1. If makeCORSPreflight is true and one of these conditions is true: + // TODO + + // 2. If request’s redirect mode is "follow", then set request’s + // service-workers mode to "none". + if (request.redirect === 'follow') { + request.serviceWorkers = 'none' } - if (controller) { - this.controller = null - controller.error(err) + // 3. Set response and actualResponse to the result of running + // HTTP-network-or-cache fetch given fetchParams. + actualResponse = response = await httpNetworkOrCacheFetch.call(this, fetchParams) + + // 4. If request’s response tainting is "cors" and a CORS check + // for request and response returns failure, then return a network error. + // TODO + + // 5. If the TAO check for request and response returns failure, then set + // request’s timing allow failed flag. + // TODO + } + + // 7. If either request’s response tainting or response’s type + // is "opaque", and the cross-origin resource policy check with + // request’s origin, request’s client, request’s destination, + // and actualResponse returns blocked, then return a network error. + // TODO + + // 8. If actualResponse’s status is a redirect status, then: + if (redirectStatus.includes(actualResponse.status)) { + // 1. If actualResponse’s status is not 303, request’s body is not null, + // and the connection uses HTTP/2, then user agents may, and are even + // encouraged to, transmit an RST_STREAM frame. + // NOTE: HTTP/2 is not supported. + + // 2. Switch on request’s redirect mode: + if (request.redirect === 'error') { + // Set response to a network error. + response = makeNetworkError() + } else if (request.redirect === 'manual') { + // Set response to an opaque-redirect filtered response whose internal + // response is actualResponse. + response = filterResponse(response, 'opaqueredirect') + } else if (request.redirect === 'follow') { + // Set response to the result of running HTTP-redirect fetch given + // fetchParams and response. + response = await httpRedirectFetch.call(this, fetchParams, response) + } else { + assert(false) } } + + // 9. Set response’s timing info to timingInfo. + // TODO + + // 10. Return response. + return response } -async function fetch (opts) { - if (opts.referrer != null) { - // TODO: Implement? - throw new NotSupportedError() +// https://fetch.spec.whatwg.org/#http-redirect-fetch +function httpRedirectFetch (fetchParams, response) { + // 1. Let request be fetchParams’s request. + const request = fetchParams.request + + // 2. Let actualResponse be response, if response is not a filtered response, + // and response’s internal response otherwise. + const actualResponse = response.internalResponse + ? response.internalResponse + : response + + // 3. Let locationURL be actualResponse’s location URL given request’s current + // URL’s fragment. + let locationURL = actualResponse.headersList.get('location') + + // 4. If locationURL is null, then return response. + if (locationURL == null) { + return response } - if (opts.referrerPolicy != null) { - // TODO: Implement? - throw new NotSupportedError() + // 5. If locationURL is failure, then return a network error. + try { + locationURL = new URL(locationURL, request.currentURL) + } catch (err) { + return makeNetworkError(err) } - if (opts.mode != null) { - // TODO: Implement? - throw new NotSupportedError() + // 6. If locationURL’s scheme is not an HTTP(S) scheme, then return a network + // error. + if (!/^https?:/.test(locationURL)) { + return makeNetworkError('URL scheme must be a HTTP(S) scheme') } - if (opts.credentials != null) { - // TODO: Implement? - throw new NotSupportedError() + // 7. If request’s redirect count is twenty, return a network error. + if (request.redirectCount === 20) { + return makeNetworkError('redirect count exceeded') } - if (opts.cache != null) { - // TODO: Implement? - throw new NotSupportedError() + // 8. Increase request’s redirect count by one. + request.redirectCount += 1 + + // 9. If request’s mode is "cors", locationURL includes credentials, and + // request’s origin is not same origin with locationURL’s origin, then return + // a network error. + if ( + request.mode === 'cors' && + request.origin !== locationURL.origin + ) { + return makeNetworkError('cross origin not allowed for request mode "cors"') } - if (opts.redirect != null) { - // TODO: Validate - } else { - opts.redirect = 'follow' + // 10. If request’s response tainting is "cors" and locationURL includes + // credentials, then return a network error. + if ( + request.responseTainting === 'cors' && + (locationURL.username || locationURL.password) + ) { + return makeNetworkError('URL cannot contain credentials for request mode "cors"') + } + + // 11. If actualResponse’s status is not 303, request’s body is non-null, + // and request’s body’s source is null, then return a network error. + if ( + actualResponse.status !== 303 && + request.body != null && + request.body.source == null + ) { + return makeNetworkError() } - if (opts.method != null) { - opts.method = normalizeAndValidateRequestMethod(opts.method) + // 12. If locationURL’s origin is not same origin with request’s current URL’s + // origin and request’s origin is not same origin with request’s current + // URL’s origin, then set request’s tainted origin flag. + if ( + locationURL.origin !== request.currentURL.origin && + request.origin !== locationURL.origin + ) { + request.taintedOrigin = true + } + + // 13. If one of the following is true + // - actualResponse’s status is 301 or 302 and request’s method is `POST` + // - actualResponse’s status is 303 and request’s method is not `GET` or `HEAD` + if ( + ([301, 302].includes(actualResponse.status) && request.method === 'POST') || + (actualResponse.status === 303 && !['GET', 'HEADER'].includes(request.method)) + ) { + // then: + // 1. Set request’s method to `GET` and request’s body to null. + request.method = 'GET' + request.body = null + + // 2. For each headerName of request-body-header name, delete headerName from + // request’s header list. + for (const headerName of requestBodyHeader) { + request.headersList.delete(headerName) + } + } + + // 14. If request’s body is non-null, then set request’s body to the first return + // value of safely extracting request’s body’s source. + if (request.body != null) { + assert(request.body.source) + request.body = safelyExtractBody(request.body.source)[0] + } + + // 15. Let timingInfo be fetchParams’s timing info. + // TODO + + // 16. Set timingInfo’s redirect end time and post-redirect start time to the + // coarsened shared current time given fetchParams’s cross-origin isolated capability. + // TODO + + // 17. If timingInfo’s redirect start time is 0, then set timingInfo’s redirect start + // time to timingInfo’s start time. + // TODO + + // 18. Append locationURL to request’s URL list. + request.urlList.push(locationURL) + request.currentURL = locationURL + + // 19. Invoke set request’s referrer policy on redirect on request and actualResponse. + // TODO + + // 20. Return the result of running main fetch given fetchParams and true. + return mainFetch.call(this, fetchParams, true) +} + +// https://fetch.spec.whatwg.org/#http-network-or-cache-fetch +async function httpNetworkOrCacheFetch ( + fetchParams, + isAuthenticationFetch = false, + isNewConnectionFetch = false +) { + // 1. Let request be fetchParams’s request. + const request = fetchParams.request + + // 2. Let httpFetchParams be null. + let httpFetchParams = null + + // 3. Let httpRequest be null. + let httpRequest = null + + // 4. Let response be null. + let response = null + + // 5. Let storedResponse be null. + // TODO + + // 6. Let httpCache be null. + const httpCache = null + + // 7. Let the revalidatingFlag be unset. + const revalidatingFlag = false + + // 8. Run these steps, but abort when the ongoing fetch is terminated: + + // 1. If request’s window is "no-window" and request’s redirect mode is + // "error", then set httpFetchParams to fetchParams and httpRequest to + // request. + if (request.window === 'no-window' && request.redirect === 'error') { + httpFetchParams = fetchParams + httpRequest = request } else { - opts.method = 'GET' + // Otherwise: + + // 1. Set httpRequest to a clone of request. + httpRequest = makeRequest(request) + + // 2. Set httpFetchParams to a copy of fetchParams. + httpFetchParams = { ...fetchParams } + + // 3. Set httpFetchParams’s request to httpRequest. + httpFetchParams.request = httpRequest } - if (opts.integrity != null) { - // TODO: Implement? - throw new NotSupportedError() + // 3. Let includeCredentials be true if one of + const includeCredentials = ( + request.credentials === 'include' || + (request.credentials === 'same-origin' && request.responseTainting === 'basic') + ) + + // 4. Let contentLength be httpRequest’s body’s length, if httpRequest’s + // body is non-null; otherwise null. + const contentLength = httpRequest.body + ? httpRequest.body.length + : null + + // 5. Let contentLengthHeaderValue be null. + let contentLengthHeaderValue = null + + // 6. If httpRequest’s body is null and httpRequest’s method is `POST` or + // `PUT`, then set contentLengthHeaderValue to `0`. + if (httpRequest.body == null && ['POST', 'PUT'].includes(httpRequest.method)) { + contentLengthHeaderValue = '0' } - if (opts.keepalive != null) { - // TODO: Validate + // 7. If contentLength is non-null, then set contentLengthHeaderValue to + // contentLength, serialized and isomorphic encoded. + if (contentLength != null) { + // TODO: isomorphic encoded + contentLengthHeaderValue = String(contentLength) } - const headers = new Headers(opts.headers) + // 8. If contentLengthHeaderValue is non-null, then append + // `Content-Length`/contentLengthHeaderValue to httpRequest’s header + // list. + if (contentLengthHeaderValue != null) { + httpRequest.headersList.append('content-length', contentLengthHeaderValue) + } - if (!headers.has('accept')) { - headers.set('accept', '*/*') + // 9. If contentLength is non-null and httpRequest’s keepalive is true, + // then: + if (contentLength != null && httpRequest.keepalive) { + // NOTE: keepalive is a noop outside of browser context. } - if (!headers.has('accept-language')) { - headers.set('accept-language', '*') + // 10 .If httpRequest’s referrer is a URL, then append + // `Referer`/httpRequest’s referrer, serialized and isomorphic encoded, + // to httpRequest’s header list. + if (httpRequest.referrer instanceof URL) { + // TODO: isomorphic encoded + httpRequest.headersList.append('referer', httpRequest.referrer.href) } - const [body, contentType] = extractBody(opts.body) + // 11. Append a request `Origin` header for httpRequest. + // TODO + + // 12. Append the Fetch metadata headers for httpRequest. [FETCH-METADATA] + // TODO - if (contentType && !headers.has('content-type')) { - headers.set('content-type', contentType) + // 13. If httpRequest’s header list does not contain `User-Agent`, then + // user agents should append `User-Agent`/default `User-Agent` value to + // httpRequest’s header list. + if (!httpRequest.headersList.has('user-agent')) { + httpRequest.headersList.append('user-agent', 'undici') } - return new Promise((resolve, reject) => this.dispatch({ - path: opts.path, - origin: opts.origin, - method: opts.method, - body: body ? (body.stream || body.source) : null, - headers: headers[kHeadersList], - maxRedirections: opts.redirect === 'follow' ? 20 : 0 // https://fetch.spec.whatwg.org/#concept-http-redirect-fetch - }, new FetchHandler(opts, (err, res) => { - if (err) { - reject(err) - } else { - resolve(res) + // 14. If httpRequest’s cache mode is "default" and httpRequest’s header + // list contains `If-Modified-Since`, `If-None-Match`, + // `If-Unmodified-Since`, `If-Match`, or `If-Range`, then set + // httpRequest’s cache mode to "no-store". + if (httpRequest.cache === 'default' && ( + httpRequest.headersList.has('if-modified-since') || + httpRequest.headersList.has('if-none-match') || + httpRequest.headersList.has('if-unmodified-since') || + httpRequest.headersList.has('if-match') || + httpRequest.headersList.has('if-range') + )) { + httpRequest.cache = 'no-store' + } + + // 15. If httpRequest’s cache mode is "no-cache", httpRequest’s prevent + // no-cache cache-control header modification flag is unset, and + // httpRequest’s header list does not contain `Cache-Control`, then append + // `Cache-Control`/`max-age=0` to httpRequest’s header list. + if ( + httpRequest.cache === 'no-cache' && + !httpRequest.preventNoCacheCacheControlHeaderModification && + !httpRequest.headersList.has('cache-control') + ) { + httpRequest.headersList.append('cache-control', 'max-age=0') + } + + // 16. If httpRequest’s cache mode is "no-store" or "reload", then: + if (httpRequest.cache === 'no-store' || httpRequest.cache === 'reload') { + // 1. If httpRequest’s header list does not contain `Pragma`, then append + // `Pragma`/`no-cache` to httpRequest’s header list. + if (!httpRequest.headersList.has('pragma')) { + httpRequest.headersList.append('pragma', 'no-cache') + } + + // 2. If httpRequest’s header list does not contain `Cache-Control`, + // then append `Cache-Control`/`no-cache` to httpRequest’s header list. + if (!httpRequest.headersList.has('cache-control')) { + httpRequest.headersList.append('cache-control', 'no-cache') + } + } + + // 17. If httpRequest’s header list contains `Range`, then append + // `Accept-Encoding`/`identity` to httpRequest’s header list. + if (httpRequest.headersList.has('range')) { + httpRequest.headersList.append('accept-encoding', 'identity') + } + + // 18. Modify httpRequest’s header list per HTTP. Do not append a given + // header if httpRequest’s header list contains that header’s name. + // TODO + + // 19. If includeCredentials is true, then: + if (includeCredentials) { + // 1. If the user agent is not configured to block cookies for httpRequest + // (see section 7 of [COOKIES]), then: + // TODO + + // 2. If httpRequest’s header list does not contain `Authorization`, then: + // TODO + } + + // 20. If there’s a proxy-authentication entry, use it as appropriate. + // TODO + + // 21. Set httpCache to the result of determining the HTTP cache + // partition, given httpRequest. + // TODO + + // 22. If httpCache is null, then set httpRequest’s cache mode to + // "no-store". + if (httpCache == null) { + httpRequest.cache = 'no-store' + } + + // 23. If httpRequest’s cache mode is neither "no-store" nor "reload", + // then: + if (httpRequest.mode !== 'no-store' && httpRequest.mode !== 'reload') { + // TODO + } + + // 9. If aborted, then: + // TODO + + // 10. If response is null, then: + if (response == null) { + // 1. If httpRequest’s cache mode is "only-if-cached", then return a + // network error. + if (httpRequest.mode === 'only-if-cached') { + return makeNetworkError('only if cached') + } + + // 2. Let forwardResponse be the result of running HTTP-network fetch + // given httpFetchParams, includeCredentials, and isNewConnectionFetch. + const forwardResponse = await httpNetworkFetch.call( + this, + httpFetchParams, + includeCredentials, + isNewConnectionFetch + ) + + // 3. If httpRequest’s method is unsafe and forwardResponse’s status is + // in the range 200 to 399, inclusive, invalidate appropriate stored + // responses in httpCache, as per the "Invalidation" chapter of HTTP + // Caching, and set storedResponse to null. [HTTP-CACHING] + if ( + !safeMethods.includes(httpRequest.method) && + (forwardResponse.status >= 200 && forwardResponse.status) + ) { + // TODO + } + + // 4. If the revalidatingFlag is set and forwardResponse’s status is 304, + // then: + if (revalidatingFlag && forwardResponse.status === 304) { + // TODO } - }))) + + // 5. If response is null, then: + if (response == null) { + // 1. Set response to forwardResponse. + response = forwardResponse + + // 2. Store httpRequest and forwardResponse in httpCache, as per the + // "Storing Responses in Caches" chapter of HTTP Caching. [HTTP-CACHING] + // TODO + } + } + + // 11. Set response’s URL list to a clone of httpRequest’s URL list. + response.urlList = [...httpRequest.urlList] + + // 12. If httpRequest’s header list contains `Range`, then set response’s + // range-requested flag. + if (httpRequest.headersList.has('range')) { + response.rangeRequested = true + } + + // 13. If response’s status is 401, httpRequest’s response tainting is not + // "cors", includeCredentials is true, and request’s window is an environment + // settings object, then: + // TODO + + // 14. If response’s status is 407, then: + if (response.status === 407) { + // TODO + } + + // 15. If all of the following are true + if ( + // response’s status is 421 + response.status === 421 && + // isNewConnectionFetch is false + !isNewConnectionFetch && + // request’s body is null, or request’s body is non-null and request’s body’s source is non-null + (request.body == null || request.body.source != null) + ) { + // 1. If the ongoing fetch is terminated, then: + if (context.terminated) { + // 1. Let aborted be the termination’s aborted flag. + const aborted = context.terminated.aborted + + // 2. If aborted is set, then return an aborted network error. + const reason = aborted ? new AbortError() : new Error('terminated') + + // 3. Return a network error. + return makeNetworkError(reason) + } + + response = await httpNetworkOrCacheFetch.call( + this, + fetchParams, + isAuthenticationFetch, + true + ) + } + + // 16. If isAuthenticationFetch is true, then create an authentication entry + // for request and the given realm. + if (isAuthenticationFetch) { + // TODO + } + + // 17. Return response. + return response } -function normalizeAndValidateRequestMethod (method) { - if (typeof method !== 'string') { - throw TypeError(`Request method: ${method} must be type 'string'`) +// https://fetch.spec.whatwg.org/#http-network-fetch +function httpNetworkFetch ( + fetchParams, + includeCredentials = false, + forceNewConnection = false +) { + // 1. Let request be fetchParams’s request. + const request = fetchParams.request + + // 2. Let response be null. + let response = null + + // 3. Let timingInfo be fetchParams’s timing info. + // TODO + + // 4. Let httpCache be the result of determining the HTTP cache partition, + // given request. + // TODO + const httpCache = null + + // 5. If httpCache is null, then set request’s cache mode to "no-store". + if (httpCache == null) { + request.cache = 'no-store' } - const normalizedMethod = method.toUpperCase() + // 6. Let networkPartitionKey be the result of determining the network + // partition key given request. + // TODO - if (METHODS.indexOf(normalizedMethod) === -1) { - throw Error(`Normalized request method: ${normalizedMethod} must be one of ${METHODS.join(', ')}`) + // ... + + let body = null + if (request.body) { + // TODO (fix): Do we need to lock the Request stream and if + // so do we need to do it immediatly? + + if (request.body.source) { + // We can bypass stream and use source directly + // since Request already should have locked the + // source stream thus making it "unusable" for + // anyone else. + body = request.body.source + } else { + body = request.body.stream + } } - return normalizedMethod + const context = this + return new Promise((resolve) => context.dispatcher.dispatch({ + path: request.currentURL.pathname + request.currentURL.search, + origin: request.currentURL.origin, + method: request.method, + body, + headers: request.headersList, + maxRedirections: 0 + }, { + onConnect (abort) { + if (context.terminated) { + abort(new AbortError()) + } else { + context.abort = () => abort(new AbortError()) + } + }, + + onHeaders (statusCode, headers, resume) { + if (statusCode < 200) { + return + } + + headers = new Headers(headers) + + if (!ReadableStream) { + ReadableStream = require('stream/web').ReadableStream + } + + const stream = statusCode === 204 + ? null + : new ReadableStream({ + async start (controller) { + context.controller = controller + }, + async pull () { + resume() + }, + async cancel (reason) { + let err + if (reason instanceof Error) { + err = reason + } else if (typeof reason === 'string') { + err = new Error(reason) + } else { + err = new AbortError() + } + + context.abort(err) + } + }, { highWaterMark: 16384 }) + + // TODO: Not sure we should use new Response here. + response = new Response(stream, { + status: statusCode, + // TODO: Get statusText from response? + statusText: STATUS_CODES[statusCode], + headers + })[kState] + + resolve(response) + + return false + }, + + onData (chunk) { + assert(context.controller) + + // Copy the Buffer to detach it from Buffer pool. + // TODO: Is this required? + chunk = new Uint8Array(chunk) + + context.controller.enqueue(chunk) + + return context.controller.desiredSize > 0 + }, + + onComplete () { + assert(context.controller) + + context.controller.close() + context.controller = null + context.abort = null + + finalizeResponse(fetchParams, response) + }, + + onError (err) { + if (context.controller) { + context.controller?.error(err) + context.controller = null + context.abort = null + } else { + // TODO: What if 204? + resolve(makeNetworkError(err)) + } + } + })) +} + +function createDeferredPromise () { + let res + let rej + const promise = new Promise((resolve, reject) => { + res = resolve + rej = reject + }) + + return { promise, resolve: res, reject: rej } } module.exports = fetch diff --git a/lib/api/api-fetch/request.js b/lib/api/api-fetch/request.js new file mode 100644 index 00000000000..9f2d93d36db --- /dev/null +++ b/lib/api/api-fetch/request.js @@ -0,0 +1,652 @@ +/* globals AbortController */ + +'use strict' + +const { METHODS } = require('http') +const { extractBody, mixinBody, cloneBody } = require('./body') +const { Headers, fill: fillHeaders, HeadersList } = require('./headers') +const util = require('../../core/util') +const { + corsSafeListedMethods, + referrerPolicy, + requestRedirect, + requestMode, + requestCredentials, + requestCache +} = require('./constants') +const { kEnumerableProperty } = util +const { + kHeaders, + kSignal, + kState, + kGuard +} = require('./symbols') +const { kHeadersList } = require('../../core/symbols') +const assert = require('assert') + +let TransformStream + +// https://fetch.spec.whatwg.org/#request-class +class Request { + // https://fetch.spec.whatwg.org/#dom-request + constructor (input, init = {}) { + if (!(input instanceof Request)) { + input = String(input) + } + + // 1. Let request be null. + let request = null + + // 2. Let fallbackMode be null. + let fallbackMode = null + + // 3. Let baseURL be this’s relevant settings object’s API base URL. + // TODO: this’s relevant settings object’s API base URL? + const baseUrl = undefined + + // 4. Let signal be null. + let signal = null + + // 5. If input is a string, then: + if (typeof input === 'string') { + // 1. Let parsedURL be the result of parsing input with baseURL. + // 2. If parsedURL is failure, then throw a TypeError. + let parsedURL + try { + parsedURL = new URL(input, baseUrl) + } catch (err) { + const error = new TypeError('Invalid URL') + error.cause = err + throw error + } + + // 3. If parsedURL includes credentials, then throw a TypeError. + if (parsedURL.username || parsedURL.password) { + throw new TypeError('Invalid URL') + } + + // 4. Set request to a new request whose URL is parsedURL. + request = makeRequest({ urlList: [parsedURL] }) + + // 5. Set fallbackMode to "cors". + fallbackMode = 'cors' + } else { + // 6. Otherwise: + + // 7. Assert: input is a Request object. + assert(input instanceof Request) + + // 8. Set request to input’s request. + request = input[kState] + + // 9. Set signal to input’s signal. + signal = input[kSignal] + } + + // 7. Let origin be this’s relevant settings object’s origin. + // TODO + + // 8. Let window be "client". + let window = 'client' + + // 9. If request’s window is an environment settings object and its origin + // is same origin with origin, then set window to request’s window. + // TODO + + // 10. If init["window"] exists and is non-null, then throw a TypeError. + if ('window' in init && window !== null) { + throw new TypeError(`'window' option '${window}' must be null`) + } + + // 11. If init["window"] exists, then set window to "no-window". + if ('window' in init) { + window = 'no-window' + + if (window !== null) { + throw new TypeError(`'window' option '${window}' must be null`) + } + } + + // 12. Set request to a new request with the following properties: + request = makeRequest({ ...request, window }) + + // 13. If init is not empty, then: + if (Object.keys(init).length > 0) { + // 1. If request’s mode is "navigate", then set it to "same-origin". + if (request.mode === 'navigate') { + request.mode = 'same-origin' + } + + // 2. Unset request’s reload-navigation flag. + request.reloadNavigation = false + + // 3. Unset request’s history-navigation flag. + request.historyNavigation = false + + // 4. Set request’s referrer to "client" + request.referrer = 'client' + + // 5. Set request’s referrer policy to the empty string. + request.referrerPolicy = '' + } + + // 14. If init["referrer"] exists, then: + if ('referrer' in init) { + // 1. Let referrer be init["referrer"]. + const referrer = init.referrer + + // 2. If referrer is the empty string, then set request’s referrer to "no-referrer". + if (!referrer === '') { + request.referrer = 'no-referrer' + } else { + // 1. Let parsedReferrer be the result of parsing referrer with + // baseURL. + // 2. If parsedReferrer is failure, then throw a TypeError. + let parsedReferrer + try { + parsedReferrer = new URL(referrer, baseUrl) + } catch (err) { + const error = new TypeError('invalid referrer') + error.cause = err + throw error + } + + // 3. If one of the following is true + // parsedReferrer’s cannot-be-a-base-URL is true, scheme is "about", + // and path contains a single string "client" + // parsedReferrer’s origin is not same origin with origin + // then set request’s referrer to "client". + // TODO + + // 4. Otherwise, set request’s referrer to parsedReferrer. + request.referrer = parsedReferrer + } + } + + // 15. If init["referrerPolicy"] exists, then set request’s referrer policy + // to it. + if ('referrerPolicy' in init) { + request.referrerPolicy = init.referrerPolicy + if (!referrerPolicy.includes(request.referrerPolicy)) { + throw new TypeError(`'referrer' option '${request.referrerPolicy}' is not a valid value of ReferrerPolicy`) + } + } + + // 16. Let mode be init["mode"] if it exists, and fallbackMode otherwise. + let mode + if ('mode' in init) { + mode = init.mode + if (!requestMode.includes(mode)) { + throw new TypeError(`'mode' option '${mode}' is not a valid value of RequestMode`) + } + } else { + mode = fallbackMode + } + + // 17. If mode is "navigate", then throw a TypeError. + if (mode === 'navigate') { + throw new TypeError() + } + + // 18. If mode is non-null, set request’s mode to mode. + if (mode != null) { + request.mode = mode + } + + // 19. If init["credentials"] exists, then set request’s credentials mode + // to it. + if ('credentials' in init) { + request.credentials = init.credentials + if (!requestCredentials.includes(request.credentials)) { + throw new TypeError(`'credentials' option '${request.credentials}' is not a valid value of RequestCredentials`) + } + } + + // 18. If init["cache"] exists, then set request’s cache mode to it. + if ('cache' in init) { + request.cache = init.cache + if (!requestCache.includes(request.cache)) { + throw new TypeError(`'cache' option '${request.cache}' is not a valid value of RequestCache`) + } + } + + // 21. If request’s cache mode is "only-if-cached" and request’s mode is + // not "same-origin", then throw a TypeError. + if (request.cache === 'only-if-cached' && request.mode !== 'same-origin') { + throw new TypeError() + } + + // 22. If init["redirect"] exists, then set request’s redirect mode to it. + if ('redirect' in init) { + request.redirect = init.redirect + if (!requestRedirect.includes(request.redirect)) { + throw new TypeError(`'redirect' option '${request.redirect}' is not a valid value of RequestRedirect`) + } + } + + // 23. If init["integrity"] exists, then set request’s integrity metadata to it. + if ('integrity' in init) { + request.integrity = init.integrity + if (typeof request.integrity !== 'string') { + throw new TypeError(`'integrity' option '${request.integrity}' is not a valid value of string`) + } + } + + // 24. If init["keepalive"] exists, then set request’s keepalive to it. + if ('keepalive' in init) { + request.keepalive = init.keepalive + if (typeof request.keepalive !== 'boolean') { + throw new TypeError(`'keepalive' option '${request.keepalive}' is not a valid value of boolean`) + } + } + + // 25. If init["method"] exists, then: + if ('method' in init) { + // 1. Let method be init["method"]. + let method = init.method + + // 2. If method is not a method or method is a forbidden method, then + // throw a TypeError. + if (typeof init.method !== 'string') { + throw TypeError(`Request method: ${init.method} must be type 'string'`) + } + + if (METHODS.indexOf(method.toUpperCase()) === -1) { + throw Error(`Normalized request init.method: ${method} must be one of ${METHODS.join(', ')}`) + } + + // 3. Normalize method. + method = init.method.toUpperCase() + + // 4. Set request’s method to method. + request.method = method + } + + // 26. If init["signal"] exists, then set signal to it. + if ('signal' in init) { + signal = init.signal + } + + // 27. Set this’s request to request. + this[kState] = request + + // 28. Set this’s signal to a new AbortSignal object with this’s relevant + // Realm. + // TODO: relevant Realm? + const ac = new AbortController() + this[kSignal] = ac.signal + + // 29. If signal is not null, then make this’s signal follow signal. + if (signal != null) { + if ( + typeof signal.aborted !== 'boolean' && + typeof signal.addEventListener !== 'function') { + throw new TypeError(`'signal' option '${signal}' is not a valid value of AbortSignal`) + } + + if (signal.aborted) { + ac.abort() + } else { + // TODO: Remove this listener on failure/success. + signal.addEventListener('abort', function () { + ac.abort() + }, { once: true }) + } + } + + // 30. Set this’s headers to a new Headers object with this’s relevant + // Realm, whose header list is request’s header list and guard is + // "request". + // TODO: relevant Realm? + this[kHeaders] = new Headers() + this[kHeaders][kGuard] = 'request' + this[kHeaders][kHeadersList] = request.headersList + + // 31. If this’s request’s mode is "no-cors", then: + if (mode === 'no-cors') { + // 1. If this’s request’s method is not a CORS-safelisted method, + // then throw a TypeError. + if (!corsSafeListedMethods.includes(request.method)) { + throw new TypeError() + } + + // 2. Set this’s headers’s guard to "request-no-cors". + this[kHeaders][kGuard] = 'request-no-cors' + } + + // 32. If init is not empty, then: + if (Object.keys(init).length !== 0) { + // 1. Let headers be a copy of this’s headers and its associated header + // list. + let headers = new Headers(this.headers) + + // 2. If init["headers"] exists, then set headers to init["headers"]. + if ('headers' in init) { + headers = init.headers + } + + // 3. Empty this’s headers’s header list. + this[kState].headersList = new HeadersList() + this[kHeaders][kHeadersList] = this[kState].headersList + + // 4. If headers is a Headers object, then for each header in its header + // list, append header’s name/header’s value to this’s headers. + // 5. Otherwise, fill this’s headers with headers. + fillHeaders(this[kHeaders], headers) + } + + // 33. Let inputBody be input’s request’s body if input is a Request + // object; otherwise null. + const inputBody = input instanceof Request ? input[kState].body : null + + // 34. If either init["body"] exists and is non-null or inputBody is + // non-null, and request’s method is `GET` or `HEAD`, then throw a + // TypeError. + if ( + (('body' in init && init != null) || inputBody != null) && + (request.method === 'GET' || request.method === 'HEAD') + ) { + throw new TypeError() + } + + // 35. Let initBody be null. + let initBody = null + + // 36. If init["body"] exists and is non-null, then: + if ('body' in init && init != null) { + // 1. Let Content-Type be null. + // 2. Set initBody and Content-Type to the result of extracting + // init["body"], with keepalive set to request’s keepalive. + const [extractedBody, contentType] = extractBody(init.body) + initBody = extractedBody + + // 3, If Content-Type is non-null and this’s headers’s header list does + // not contain `Content-Type`, then append `Content-Type`/Content-Type to + // this’s headers. + if (contentType && !this[kHeaders].has('content-type')) { + this[kHeaders].append('content-type', contentType) + } + } + + // 37. Let inputOrInitBody be initBody if it is non-null; otherwise + // inputBody. + const inputOrInitBody = initBody ?? inputBody + + // 38. If inputOrInitBody is non-null and inputOrInitBody’s source is + // null, then: + if (inputOrInitBody != null && inputOrInitBody.source == null) { + // 1. If this’s request’s mode is neither "same-origin" nor "cors", + // then throw a TypeError. + if (request.mode !== 'same-origin' && request.mode !== 'cors') { + throw new TypeError() + } + + // 2. Set this’s request’s use-CORS-preflight flag. + request.useCORSPreflightFlag = true + } + + // 39. Let finalBody be inputOrInitBody. + let finalBody = inputOrInitBody + + // 40. If initBody is null and inputBody is non-null, then: + if (initBody == null && inputBody != null) { + // 1. If input is unusable, then throw a TypeError. + if (util.isDisturbed(inputBody.stream) || inputBody.stream.locked) { + throw new TypeError('unusable') + } + + // 2. Set finalBody to the result of creating a proxy for inputBody. + if (!TransformStream) { + TransformStream = require('stream/web').TransformStream + } + + // https://streams.spec.whatwg.org/#readablestream-create-a-proxy + const identityTransform = new TransformStream() + inputBody.stream.pipeThrough(identityTransform) + finalBody = { + source: inputBody.source, + length: inputBody.length, + stream: identityTransform.readable + } + } + + // 41. Set this’s request’s body to finalBody. + this[kState].body = finalBody + } + + get [Symbol.toStringTag] () { + return this.constructor.name + } + + toString () { + return Object.prototype.toString.call(this) + } + + // Returns request’s HTTP method, which is "GET" by default. + get method () { + // The method getter steps are to return this’s request’s method. + return this[kState].method + } + + // Returns the URL of request as a string. + get url () { + // The url getter steps are to return this’s request’s URL, serialized. + return this[kState].url.toString() + } + + // Returns a Headers object consisting of the headers associated with request. + // Note that headers added in the network layer by the user agent will not + // be accounted for in this object, e.g., the "Host" header. + get headers () { + // The headers getter steps are to return this’s headers. + return this[kHeaders] + } + + // Returns the kind of resource requested by request, e.g., "document" + // or "script". + get destination () { + // The destination getter are to return this’s request’s destination. + return '' + } + + // Returns the referrer of request. Its value can be a same-origin URL if + // explicitly set in init, the empty string to indicate no referrer, and + // "about:client" when defaulting to the global’s default. This is used + // during fetching to determine the value of the `Referer` header of the + // request being made. + get referrer () { + // 1. If this’s request’s referrer is "no-referrer", then return the + // empty string. + if (this[kState].referrer === 'no-referrer') { + return '' + } + + // 2. If this’s request’s referrer is "client", then return + // "about:client". + if (this[kState].referrer === 'client') { + return 'about:client' + } + + // Return this’s request’s referrer, serialized. + return this[kState].referrer.toString() + } + + // Returns the referrer policy associated with request. + // This is used during fetching to compute the value of the request’s + // referrer. + get referrerPolicy () { + // The referrerPolicy getter steps are to return this’s request’s referrer policy. + return this[kState].referrerPolicy + } + + // Returns the mode associated with request, which is a string indicating + // whether the request will use CORS, or will be restricted to same-origin + // URLs. + get mode () { + // The mode getter steps are to return this’s request’s mode. + return this[kState].mode + } + + // Returns the credentials mode associated with request, + // which is a string indicating whether credentials will be sent with the + // request always, never, or only when sent to a same-origin URL. + get credentials () { + // The credentials getter steps are to return this’s request’s credentials mode. + return this[kState].credentials + } + + // Returns the cache mode associated with request, + // which is a string indicating how the request will + // interact with the browser’s cache when fetching. + get cache () { + // The cache getter steps are to return this’s request’s cache mode. + return this[kState].cache + } + + // Returns the redirect mode associated with request, + // which is a string indicating how redirects for the + // request will be handled during fetching. A request + // will follow redirects by default. + get redirect () { + // The redirect getter steps are to return this’s request’s redirect mode. + return this[kState].redirect + } + + // Returns request’s subresource integrity metadata, which is a + // cryptographic hash of the resource being fetched. Its value + // consists of multiple hashes separated by whitespace. [SRI] + get integrity () { + // The integrity getter steps are to return this’s request’s integrity + // metadata. + return this[kState].integrity + } + + // Returns a boolean indicating whether or not request can outlive the + // global in which it was created. + get keepalive () { + // The keepalive getter steps are to return this’s request’s keepalive. + return this[kState].keepalive + } + + // Returns a boolean indicating whether or not request is for a reload + // navigation. + get isReloadNavigation () { + // The isReloadNavigation getter steps are to return true if this’s + // request’s reload-navigation flag is set; otherwise false. + return this[kState].reloadNavigation + } + + // Returns a boolean indicating whether or not request is for a history + // navigation (a.k.a. back-foward navigation). + get isHistoryNavigation () { + // The isHistoryNavigation getter steps are to return true if this’s request’s + // history-navigation flag is set; otherwise false. + return this[kState].historyNavigation + } + + // Returns the signal associated with request, which is an AbortSignal + // object indicating whether or not request has been aborted, and its + // abort event handler. + get signal () { + // The signal getter steps are to return this’s signal. + return this[kSignal] + } + + // Returns a clone of request. + clone () { + // 1. If this is unusable, then throw a TypeError. + if (this.bodyUsed || (this.body && this.body.locked)) { + throw new TypeError() + } + + // 2. Let clonedRequest be the result of cloning this’s request. + const clonedRequest = makeRequest(this[kState]) + clonedRequest.body = cloneBody(clonedRequest.body) + + // 3. Let clonedRequestObject be the result of creating a Request object, + // given clonedRequest, this’s headers’s guard, and this’s relevant Realm. + // TODO: relevant Realm? + const clonedRequestObject = new Request() + clonedRequestObject[kState] = clonedRequest + clonedRequestObject[kHeaders][kHeadersList] = clonedRequest.headersList + clonedRequestObject[kHeaders][kGuard] = this[kHeaders][kGuard] + + // 4. Make clonedRequestObject’s signal follow this’s signal. + const ac = new AbortController() + if (this.signal.aborted) { + ac.abort() + } else { + this.signal.addEventListener('abort', function () { + ac.abort() + }, { once: true }) + } + clonedRequestObject[kSignal] = ac.signal + + // 4. Return clonedRequestObject. + return clonedRequestObject + } +} + +mixinBody(Request.prototype) + +function makeRequest (init) { + // https://fetch.spec.whatwg.org/#requests + const request = { + method: 'GET', + localURLsOnly: false, + unsafeRequest: false, + body: null, + client: null, + reservedClient: null, + replacesClientId: '', + window: 'client', + keepalive: false, + serviceWorkers: 'all', + initiator: '', + destination: '', + priority: null, + origin: 'client', + policyContainer: 'client', + referrer: 'client', + referrerPolicy: '', + mode: 'no-cors', + useCORSPreflightFlag: false, + credentials: 'same-origin', + useCredentials: false, + cache: 'default', + redirect: 'follow', + integrity: '', + cryptoGraphicsNonceMetadata: '', + parserMetadata: '', + reloadNavigation: false, + historyNavigation: false, + userActivation: false, + taintedOrigin: false, + redirectCount: 0, + responseTainting: 'basic', + preventNoCacheCacheControlHeaderModification: false, + done: false, + timingAllowFailed: false, + ...init, + headersList: init.headersList + ? new HeadersList(...init.headersList) + : new HeadersList(), + urlList: init.urlList + ? [...init.urlList.map(url => new URL(url))] + : [] + } + request.currentURL = request.urlList[request.urlList.length - 1] + request.url = request.urlList[0] + return request +} + +Object.defineProperties(Request.prototype, { + method: kEnumerableProperty, + url: kEnumerableProperty, + headers: kEnumerableProperty, + redirect: kEnumerableProperty, + clone: kEnumerableProperty, + signal: kEnumerableProperty +}) + +module.exports = { Request, makeRequest } diff --git a/lib/api/api-fetch/response.js b/lib/api/api-fetch/response.js index db36102d658..855a6ee8ed9 100644 --- a/lib/api/api-fetch/response.js +++ b/lib/api/api-fetch/response.js @@ -1,141 +1,348 @@ 'use strict' -const Headers = require('./headers') -const { Blob } = require('buffer') -const { STATUS_CODES } = require('http') -const { - NotSupportedError -} = require('../../core/errors') +const { Headers, HeadersList, fill } = require('./headers') +const { extractBody, cloneBody, mixinBody } = require('./body') const util = require('../../core/util') - +const { kEnumerableProperty } = util +const { redirectStatus, nullBodyStatus, forbiddenHeaderNames } = require('./constants') +const assert = require('assert') const { - kType, - kStatus, - kStatusText, - kUrlList, + kState, kHeaders, - kBody + kGuard } = require('./symbols') +const { kHeadersList } = require('../../core/symbols') +// https://fetch.spec.whatwg.org/#response-class class Response { - constructor ({ - type, - url, - body, - statusCode, - headers, - context - }) { - this[kType] = type || 'default' - this[kStatus] = statusCode || 0 - this[kStatusText] = STATUS_CODES[statusCode] || '' - this[kUrlList] = Array.isArray(url) ? url : (url ? [url] : []) - this[kHeaders] = headers || new Headers() - this[kBody] = body || null - - if (context && context.history) { - this[kUrlList].push(...context.history) + // Creates network error Response. + static error () { + // The static error() method steps are to return the result of creating a + // Response object, given a new network error, "immutable", and this’s + // relevant Realm. + // TODO: relevant Realm? + const responseObject = new Response() + responseObject[kState] = makeNetworkError() + responseObject[kHeaders][kHeadersList] = responseObject[kState].headersList + responseObject[kHeaders][kGuard] = 'immutable' + return responseObject + } + + // Creates a redirect Response that redirects to url with status status. + static redirect (url, status = 302) { + // 1. Let parsedURL be the result of parsing url with current settings + // object’s API base URL. + // 2. If parsedURL is failure, then throw a TypeError. + // TODO: base-URL? + let parsedURL + try { + parsedURL = new URL(url) + } catch (err) { + const error = new TypeError() + error.cause = err + throw error } + + // 3. If status is not a redirect status, then throw a RangeError. + if (!redirectStatus.includes(status)) { + throw new RangeError() + } + + // 4. Let responseObject be the result of creating a Response object, + // given a new response, "immutable", and this’s relevant Realm. + // TODO: relevant Realm? + const responseObject = new Response() + responseObject[kHeaders][kGuard] = 'immutable' + + // 5. Set responseObject’s response’s status to status. + responseObject[kState].status = status + + // 6. Let value be parsedURL, serialized and isomorphic encoded. + // TODO: isomorphic encoded? + const value = parsedURL.toString() + + // 7. Append `Location`/value to responseObject’s response’s header list. + responseObject[kState].headersList.push('location', value) + + // 8. Return responseObject. + return responseObject } + // https://fetch.spec.whatwg.org/#dom-response + constructor (body = null, init = {}) { + // 1. If init["status"] is not in the range 200 to 599, inclusive, then + // throw a RangeError. + if ('status' in init) { + if (!Number.isFinite(init.status)) { + throw new TypeError() + } + + if (init.status < 200 || init.status > 599) { + throw new RangeError() + } + } + + if ('statusText' in init) { + if (typeof init.statusText !== 'string') { + throw new TypeError() + } + + // 2. If init["statusText"] does not match the reason-phrase token + // production, then throw a TypeError. + // See, https://datatracker.ietf.org/doc/html/rfc7230#section-3.1.2: + // reason-phrase = *( HTAB / SP / VCHAR / obs-text ) + // TODO + } + + // 3. Set this’s response to a new response. + this[kState] = makeResponse(init) + + // 4. Set this’s headers to a new Headers object with this’s relevant + // Realm, whose header list is this’s response’s header list and guard + // is "response". + // TODO: relevant Realm? + this[kHeaders] = new Headers() + this[kHeaders][kGuard] = 'response' + this[kHeaders][kHeadersList] = this[kState].headersList + + // 5. Set this’s response’s status to init["status"]. + if ('status' in init) { + this[kState].status = init.status + } + + // 6. Set this’s response’s status message to init["statusText"]. + if ('statusText' in init) { + this[kState].statusText = init.statusText + } + + // 7. If init["headers"] exists, then fill this’s headers with init["headers"]. + if ('headers' in init) { + fill(this[kHeaders], init.headers) + } + + // 8. If body is non-null, then: + if (body != null) { + // 1. If init["status"] is a null body status, then throw a TypeError. + if (nullBodyStatus.includes(init.status)) { + throw new TypeError() + } + + // 2. Let Content-Type be null. + // 3. Set this’s response’s body and Content-Type to the result of + // extracting body. + const [extractedBody, contentType] = extractBody(body) + this[kState].body = extractedBody + + // 4. If Content-Type is non-null and this’s response’s header list does + // not contain `Content-Type`, then append `Content-Type`/Content-Type + // to this’s response’s header list. + if (contentType && !this.headers.has('content-type')) { + this.headers.set('content-type', contentType) + } + } + } + + get [Symbol.toStringTag] () { + return this.constructor.name + } + + toString () { + return Object.prototype.toString.call(this) + } + + // Returns response’s type, e.g., "cors". get type () { - return this[kType] + // The type getter steps are to return this’s response’s type. + return this[kState].type } + // Returns response’s URL, if it has one; otherwise the empty string. get url () { - const length = this[kUrlList].length - return length === 0 ? '' : this[kUrlList][length - 1].toString() + // The url getter steps are to return the empty string if this’s + // response’s URL is null; otherwise this’s response’s URL, + // serialized with exclude fragment set to true. + let url + + // https://fetch.spec.whatwg.org/#responses + // A response has an associated URL. It is a pointer to the last URL + // in response’s URL list and null if response’s URL list is empty. + const urlList = this[kState].urlList + const length = urlList.length + url = length === 0 ? null : urlList[length - 1].toString() + + if (url == null) { + return '' + } + + if (url.hash) { + url = new URL(url) + url.hash = '' + } + + return url.toString() } + // Returns whether response was obtained through a redirect. get redirected () { - return this[kUrlList].length > 1 + // The redirected getter steps are to return true if this’s response’s URL + // list has more than one item; otherwise false. + return this[kState].urlList.length > 1 } + // Returns response’s status. get status () { - return this[kStatus] + // The status getter steps are to return this’s response’s status. + return this[kState].status } + // Returns whether response’s status is an ok status. get ok () { - return this[kStatus] >= 200 && this[kStatus] <= 299 + // The ok getter steps are to return true if this’s response’s status is an + // ok status; otherwise false. + return this[kState].status >= 200 && this[kState].status <= 299 } + // Returns response’s status message. get statusText () { - return this[kStatusText] + // The statusText getter steps are to return this’s response’s status + // message. + return this[kState].statusText } + // Returns response’s headers as Headers. get headers () { + // The headers getter steps are to return this’s headers. return this[kHeaders] } - async blob () { - const chunks = [] - if (this.body) { - if (this.bodyUsed || this.body.locked) { - throw new TypeError('unusable') - } - - for await (const chunk of this.body) { - chunks.push(chunk) - } + // Returns a clone of response. + clone () { + // 1. If this is unusable, then throw a TypeError. + if (this.bodyUsed || (this.body && this.body.locked)) { + throw new TypeError() } - return new Blob(chunks, { type: this.headers.get('Content-Type') || '' }) - } - - async arrayBuffer () { - const blob = await this.blob() - return await blob.arrayBuffer() - } + // 2. Let clonedResponse be the result of cloning this’s response. + const clonedResponse = makeResponse(this[kState]) + clonedResponse.body = cloneBody(clonedResponse.body) - async text () { - const blob = await this.blob() - return await blob.text() - } + // 3. Return the result of creating a Response object, given + // clonedResponse, this’s headers’s guard, and this’s relevant Realm. + // TODO: relevant Realm? + const clonedResponseObject = new Response() + clonedResponseObject[kState] = clonedResponse + clonedResponseObject[kHeaders][kHeadersList] = clonedResponse.headersList + clonedResponseObject[kHeaders][kGuard] = this[kHeaders][kGuard] - async json () { - return JSON.parse(await this.text()) + return clonedResponseObject } +} +mixinBody(Response.prototype) - async formData () { - // TODO: Implement. - throw new NotSupportedError('formData') - } +Object.defineProperties(Response.prototype, { + type: kEnumerableProperty, + url: kEnumerableProperty, + status: kEnumerableProperty, + ok: kEnumerableProperty, + redirected: kEnumerableProperty, + statusText: kEnumerableProperty, + headers: kEnumerableProperty, + clone: kEnumerableProperty +}) - get body () { - return this[kBody] +function makeResponse (init) { + return { + internalResponse: null, + aborted: false, + rangeRequested: false, + type: 'default', + status: 200, + statusText: '', + ...init, + headersList: init.headersList + ? new HeadersList(...init.headersList) + : new HeadersList(), + urlList: init.urlList + ? [...init.urlList.map(url => new URL(url))] + : [] } +} - get bodyUsed () { - return util.isDisturbed(this.body) - } +function makeNetworkError (reason) { + return makeResponse({ + type: 'error', + status: 0, + error: reason instanceof Error + ? reason + : new Error(reason ? String(reason) : reason), + aborted: reason && reason.name === 'AbortError' + }) +} - clone () { - let body = null +function filterResponse (response, type) { + // Set response to the following filtered response with response as its + // internal response, depending on request’s response tainting: + if (type === 'basic') { + // A basic filtered response is a filtered response whose type is "basic" + // and header list excludes any headers in internal response’s header list + // whose name is a forbidden response-header name. - if (this[kBody]) { - if (util.isDisturbed(this[kBody])) { - throw new TypeError('disturbed') + const headers = [] + for (let n = 0; n < response.headersList.length; n += 2) { + if (!forbiddenHeaderNames.includes(response.headersList[n])) { + headers.push(response.headersList[n + 0], response.headersList[n + 1]) } + } - if (this[kBody].locked) { - throw new TypeError('locked') - } + return makeResponse({ + ...response, + internalResponse: response, + headersList: new HeadersList(...headers), + type: 'basic' + }) + } else if (type === 'cors') { + // A CORS filtered response is a filtered response whose type is "cors" + // and header list excludes any headers in internal response’s header + // list whose name is not a CORS-safelisted response-header name, given + // internal response’s CORS-exposed header-name list. - // https://fetch.spec.whatwg.org/#concept-body-clone - const [out1, out2] = this[kBody].tee() + // TODO: This is not correct... + return makeResponse({ + ...response, + internalResponse: response, + type: 'cors' + }) + } else if (type === 'opaque') { + // An opaque filtered response is a filtered response whose type is + // "opaque", URL list is the empty list, status is 0, status message + // is the empty byte sequence, header list is empty, and body is null. - this[kBody] = out1 - body = out2 - } + return makeResponse({ + ...response, + internalResponse: response, + type: 'opaque', + urlList: [], + status: 0, + statusText: '', + body: null + }) + } else if (type === 'opaqueredirect') { + // An opaque-redirect filtered response is a filtered response whose type + // is "opaqueredirect", status is 0, status message is the empty byte + // sequence, header list is empty, and body is null. - return new Response({ - type: this[kType], - statusCode: this[kStatus], - url: this[kUrlList], - headers: this[kHeaders], - body + return makeResponse({ + ...response, + internalResponse: response, + type: 'opaque', + status: 0, + statusText: '', + headersList: new HeadersList(), + body: null }) + } else { + assert(false) } } -module.exports = Response +module.exports = { makeNetworkError, makeResponse, filterResponse, Response } diff --git a/lib/api/api-fetch/symbols.js b/lib/api/api-fetch/symbols.js index 7b45f11a9e0..23580d99b1d 100644 --- a/lib/api/api-fetch/symbols.js +++ b/lib/api/api-fetch/symbols.js @@ -1,11 +1,9 @@ 'use strict' module.exports = { - kType: Symbol('type'), - kStatus: Symbol('status'), - kStatusText: Symbol('status text'), - kUrlList: Symbol('url list'), + kUrl: Symbol('url'), kHeaders: Symbol('headers'), - kBody: Symbol('body'), - kBodyUsed: Symbol('body used') + kSignal: Symbol('signal'), + kState: Symbol('state'), + kGuard: Symbol('guard') } diff --git a/lib/api/api-fetch/util.js b/lib/api/api-fetch/util.js new file mode 100644 index 00000000000..749a072e45e --- /dev/null +++ b/lib/api/api-fetch/util.js @@ -0,0 +1,100 @@ +'use strict' + +const badPorts = [ + 1, + 7, + 9, + 11, + 13, + 15, + 17, + 19, + 20, + 21, + 22, + 23, + 25, + 37, + 42, + 43, + 53, + 69, + 77, + 79, + 87, + 95, + 101, + 102, + 103, + 104, + 109, + 110, + 111, + 113, + 115, + 117, + 119, + 123, + 135, + 137, + 139, + 143, + 161, + 179, + 389, + 427, + 465, + 512, + 513, + 514, + 515, + 526, + 530, + 531, + 532, + 540, + 548, + 554, + 556, + 563, + 587, + 601, + 636, + 989, + 990, + 993, + 995, + 1719, + 1720, + 1723, + 2049, + 3659, + 4045, + 5060, + 5061, + 6000, + 6566, + 6665, + 6666, + 6667, + 6668, + 6669, + 6697, + 10080 +] + +module.exports = { + badPort (request) { + // 1. Let url be request’s current URL. + const url = request.currentURL + + // 2. If url’s scheme is an HTTP(S) scheme and url’s port is a bad port, + // then return blocked. + if (/^http?s/.test(url.protocol) && badPorts.includes(url.port)) { + return 'blocked' + } + + // 3. Return allowed. + return 'allowed' + } +} diff --git a/lib/api/index.js b/lib/api/index.js index 6e957a8bf73..046f5c3bed4 100644 --- a/lib/api/index.js +++ b/lib/api/index.js @@ -10,4 +10,7 @@ module.exports.connect = require('./api-connect') if (nodeMajor >= 16) { module.exports.fetch = require('./api-fetch') + module.exports.fetch.Headers = require('./api-fetch/headers').Headers + module.exports.fetch.Response = require('./api-fetch/response').Response + module.exports.fetch.Request = require('./api-fetch/request').Request } diff --git a/lib/api/readable.js b/lib/api/readable.js index cd97b1e5b43..d4efeec95b9 100644 --- a/lib/api/readable.js +++ b/lib/api/readable.js @@ -130,7 +130,7 @@ module.exports = class BodyReadable extends Readable { // https://fetch.spec.whatwg.org/#dom-body-body get body () { if (!this[kBody]) { - this[kBody] = Readable.toWeb(this) + this[kBody] = util.toWeb(this) if (this[kConsume]) { // TODO: Is this the best way to force a lock? this[kBody].getReader() // Ensure stream is locked. diff --git a/lib/core/errors.js b/lib/core/errors.js index 8e9b13b46c5..7f7ac4a958a 100644 --- a/lib/core/errors.js +++ b/lib/core/errors.js @@ -1,5 +1,13 @@ 'use strict' +class AbortError extends Error { + constructor () { + super('The operation was aborted') + this.code = 'ABORT_ERR' + this.name = 'AbortError' + } +} + class UndiciError extends Error { constructor (message) { super(message) @@ -196,6 +204,7 @@ class InvalidThisError extends TypeError { } module.exports = { + AbortError, HTTPParserError, UndiciError, HeadersTimeoutError, diff --git a/lib/core/request.js b/lib/core/request.js index 33552ae83dd..f242930c1b9 100644 --- a/lib/core/request.js +++ b/lib/core/request.js @@ -52,6 +52,9 @@ class Request { this.body = null } else if (util.isStream(body)) { this.body = body + } else if (body instanceof DataView) { + // TODO: Why is DataView special? + this.body = body.buffer.byteLength ? Buffer.from(body.buffer) : null } else if (body instanceof ArrayBuffer || ArrayBuffer.isView(body)) { this.body = body.byteLength ? Buffer.from(body) : null } else if (util.isBuffer(body)) { diff --git a/lib/core/util.js b/lib/core/util.js index 845247c321b..57d614f9457 100644 --- a/lib/core/util.js +++ b/lib/core/util.js @@ -239,7 +239,11 @@ function isDisturbed (body) { )) } +const kEnumerableProperty = Object.create(null) +kEnumerableProperty.enumerable = true + module.exports = { + kEnumerableProperty, nop, isDisturbed, isAborted, diff --git a/package.json b/package.json index b89098adc78..8689f11b2c5 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,10 @@ "build:wasm": "node build/wasm.js --docker", "lint": "standard | snazzy", "lint:fix": "standard --fix | snazzy", - "test": "tap test/*.js --no-coverage && jest test/jest/test", + "test": "tap test/*.js --no-coverage && mocha test/node-fetch && jest test/jest/test", + "test:node-fetch": "mocha test/node-fetch", + "test:jest": "jest test/jest/test", + "test:tap": "tap test/*.js --no-coverage ", "test:tdd": "tap test/*.js -w --no-coverage-report", "test:typescript": "tsd", "coverage": "standard | snazzy && tap test/*.js", @@ -50,13 +53,21 @@ "@sinonjs/fake-timers": "^7.0.5", "@types/node": "^15.0.2", "abort-controller": "^3.0.0", + "busboy": "^0.3.1", + "chai": "^4.3.4", + "chai-as-promised": "^7.1.1", + "chai-iterator": "^3.0.2", + "chai-string": "^1.5.0", "concurrently": "^6.1.0", "cronometro": "^0.8.0", + "delay": "^5.0.0", "docsify-cli": "^4.4.2", "https-pem": "^2.0.0", "husky": "^6.0.0", "jest": "^27.0.5", "jsfuzz": "^1.0.15", + "mocha": "^9.0.3", + "p-timeout": "^3.2.0", "pre-commit": "^1.2.2", "proxy": "^1.0.2", "proxyquire": "^2.1.3", @@ -72,6 +83,9 @@ "node": ">=12.18" }, "standard": { + "env": [ + "mocha" + ], "ignore": [ "lib/llhttp/constants.js", "lib/llhttp/utils.js" diff --git a/test/client-fetch.js b/test/client-fetch.js index d15c9e0d849..702d0560064 100644 --- a/test/client-fetch.js +++ b/test/client-fetch.js @@ -1,24 +1,32 @@ 'use strict' const { test } = require('tap') -const { fetch } = require('..') -const { createServer } = require('http') +const { createServer } = require('https') const nodeMajor = Number(process.versions.node.split('.')[0]) +const pem = require('https-pem') test('fetch', { skip: nodeMajor < 16 }, t => { + const { fetch, setGlobalDispatcher, Agent } = require('..') + + setGlobalDispatcher(new Agent({ + connect: { + rejectUnauthorized: false + } + })) + t.test('request json', (t) => { t.plan(1) const obj = { asd: true } - const server = createServer((req, res) => { + const server = createServer(pem, (req, res) => { res.end(JSON.stringify(obj)) }) t.teardown(server.close.bind(server)) server.listen(0, async () => { - const body = await fetch(`http://localhost:${server.address().port}`) + const body = await fetch(`https://localhost:${server.address().port}`) t.strictSame(obj, await body.json()) }) }) @@ -27,13 +35,13 @@ test('fetch', { t.plan(1) const obj = { asd: true } - const server = createServer((req, res) => { + const server = createServer(pem, (req, res) => { res.end(JSON.stringify(obj)) }) t.teardown(server.close.bind(server)) server.listen(0, async () => { - const body = await fetch(`http://localhost:${server.address().port}`) + const body = await fetch(`https://localhost:${server.address().port}`) t.strictSame(JSON.stringify(obj), await body.text()) }) }) @@ -42,13 +50,13 @@ test('fetch', { t.plan(1) const obj = { asd: true } - const server = createServer((req, res) => { + const server = createServer(pem, (req, res) => { res.end(JSON.stringify(obj)) }) t.teardown(server.close.bind(server)) server.listen(0, async () => { - const body = await fetch(`http://localhost:${server.address().port}`) + const body = await fetch(`https://localhost:${server.address().port}`) t.strictSame(Buffer.from(JSON.stringify(obj)), Buffer.from(await body.arrayBuffer())) }) }) @@ -57,14 +65,14 @@ test('fetch', { t.plan(1) const obj = { asd: true } - const server = createServer((req, res) => { + const server = createServer(pem, (req, res) => { res.setHeader('Content-Type', 'application/json') res.end(JSON.stringify(obj)) }) t.teardown(server.close.bind(server)) server.listen(0, async () => { - const response = await fetch(`http://localhost:${server.address().port}`) + const response = await fetch(`https://localhost:${server.address().port}`) t.equal('application/json', (await response.blob()).type) }) }) diff --git a/test/node-fetch/LICENSE b/test/node-fetch/LICENSE new file mode 100644 index 00000000000..41ca1b6eb4d --- /dev/null +++ b/test/node-fetch/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2016 - 2020 Node Fetch Team + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/test/node-fetch/headers.js b/test/node-fetch/headers.js new file mode 100644 index 00000000000..44f9bcc8f74 --- /dev/null +++ b/test/node-fetch/headers.js @@ -0,0 +1,282 @@ +/* eslint no-unused-expressions: "off" */ + +const { format } = require('util') +const chai = require('chai') +const chaiIterator = require('chai-iterator') +const { Headers } = require('../../lib/api/api-fetch/headers.js') + +chai.use(chaiIterator) + +const { expect } = chai + +describe('Headers', () => { + it('should have attributes conforming to Web IDL', () => { + const headers = new Headers() + expect(Object.getOwnPropertyNames(headers)).to.be.empty + const enumerableProperties = [] + + for (const property in headers) { + enumerableProperties.push(property) + } + + for (const toCheck of [ + 'append', + 'delete', + 'entries', + 'forEach', + 'get', + 'has', + 'keys', + 'set', + 'values' + ]) { + expect(enumerableProperties).to.contain(toCheck) + } + }) + + it('should allow iterating through all headers with forEach', () => { + const headers = new Headers([ + ['b', '2'], + ['c', '4'], + ['b', '3'], + ['a', '1'] + ]) + expect(headers).to.have.property('forEach') + + const result = [] + for (const [key, value] of headers.entries()) { + result.push([key, value]) + } + + expect(result).to.deep.equal([ + ['a', '1'], + ['b', '2, 3'], + ['c', '4'] + ]) + }) + + it('should be iterable with forEach', () => { + const headers = new Headers() + headers.append('Accept', 'application/json') + headers.append('Accept', 'text/plain') + headers.append('Content-Type', 'text/html') + + const results = [] + headers.forEach((value, key, object) => { + results.push({ value, key, object }) + }) + + expect(results.length).to.equal(2) + expect({ key: 'accept', value: 'application/json, text/plain', object: headers }).to.deep.equal(results[0]) + expect({ key: 'content-type', value: 'text/html', object: headers }).to.deep.equal(results[1]) + }) + + xit('should set "this" to undefined by default on forEach', () => { + const headers = new Headers({ Accept: 'application/json' }) + headers.forEach(function () { + expect(this).to.be.undefined + }) + }) + + it('should accept thisArg as a second argument for forEach', () => { + const headers = new Headers({ Accept: 'application/json' }) + const thisArg = {} + headers.forEach(function () { + expect(this).to.equal(thisArg) + }, thisArg) + }) + + it('should allow iterating through all headers with for-of loop', () => { + const headers = new Headers([ + ['b', '2'], + ['c', '4'], + ['a', '1'] + ]) + headers.append('b', '3') + expect(headers).to.be.iterable + + const result = [] + for (const pair of headers) { + result.push(pair) + } + + expect(result).to.deep.equal([ + ['a', '1'], + ['b', '2, 3'], + ['c', '4'] + ]) + }) + + it('should allow iterating through all headers with entries()', () => { + const headers = new Headers([ + ['b', '2'], + ['c', '4'], + ['a', '1'] + ]) + headers.append('b', '3') + + expect(headers.entries()).to.be.iterable + .and.to.deep.iterate.over([ + ['a', '1'], + ['b', '2, 3'], + ['c', '4'] + ]) + }) + + it('should allow iterating through all headers with keys()', () => { + const headers = new Headers([ + ['b', '2'], + ['c', '4'], + ['a', '1'] + ]) + headers.append('b', '3') + + expect(headers.keys()).to.be.iterable + .and.to.iterate.over(['a', 'b', 'c']) + }) + + it('should allow iterating through all headers with values()', () => { + const headers = new Headers([ + ['b', '2'], + ['c', '4'], + ['a', '1'] + ]) + headers.append('b', '3') + + expect(headers.values()).to.be.iterable + .and.to.iterate.over(['1', '2, 3', '4']) + }) + + it('should reject illegal header', () => { + const headers = new Headers() + expect(() => new Headers({ 'He y': 'ok' })).to.throw(TypeError) + expect(() => new Headers({ 'Hé-y': 'ok' })).to.throw(TypeError) + expect(() => new Headers({ 'He-y': 'ăk' })).to.throw(TypeError) + expect(() => headers.append('Hé-y', 'ok')).to.throw(TypeError) + expect(() => headers.delete('Hé-y')).to.throw(TypeError) + expect(() => headers.get('Hé-y')).to.throw(TypeError) + expect(() => headers.has('Hé-y')).to.throw(TypeError) + expect(() => headers.set('Hé-y', 'ok')).to.throw(TypeError) + // Should reject empty header + expect(() => headers.append('', 'ok')).to.throw(TypeError) + }) + + xit('should ignore unsupported attributes while reading headers', () => { + const FakeHeader = function () {} + // Prototypes are currently ignored + // This might change in the future: #181 + FakeHeader.prototype.z = 'fake' + + const res = new FakeHeader() + res.a = 'string' + res.b = ['1', '2'] + res.c = '' + res.d = [] + res.e = 1 + res.f = [1, 2] + res.g = { a: 1 } + res.h = undefined + res.i = null + res.j = Number.NaN + res.k = true + res.l = false + res.m = Buffer.from('test') + + const h1 = new Headers(res) + h1.set('n', [1, 2]) + h1.append('n', ['3', 4]) + + const h1Raw = h1.raw() + + expect(h1Raw.a).to.include('string') + expect(h1Raw.b).to.include('1,2') + expect(h1Raw.c).to.include('') + expect(h1Raw.d).to.include('') + expect(h1Raw.e).to.include('1') + expect(h1Raw.f).to.include('1,2') + expect(h1Raw.g).to.include('[object Object]') + expect(h1Raw.h).to.include('undefined') + expect(h1Raw.i).to.include('null') + expect(h1Raw.j).to.include('NaN') + expect(h1Raw.k).to.include('true') + expect(h1Raw.l).to.include('false') + expect(h1Raw.m).to.include('test') + expect(h1Raw.n).to.include('1,2') + expect(h1Raw.n).to.include('3,4') + + expect(h1Raw.z).to.be.undefined + }) + + xit('should wrap headers', () => { + const h1 = new Headers({ + a: '1' + }) + const h1Raw = h1.raw() + + const h2 = new Headers(h1) + h2.set('b', '1') + const h2Raw = h2.raw() + + const h3 = new Headers(h2) + h3.append('a', '2') + const h3Raw = h3.raw() + + expect(h1Raw.a).to.include('1') + expect(h1Raw.a).to.not.include('2') + + expect(h2Raw.a).to.include('1') + expect(h2Raw.a).to.not.include('2') + expect(h2Raw.b).to.include('1') + + expect(h3Raw.a).to.include('1') + expect(h3Raw.a).to.include('2') + expect(h3Raw.b).to.include('1') + }) + + xit('should accept headers as an iterable of tuples', () => { + let headers + + headers = new Headers([ + ['a', '1'], + ['b', '2'], + ['a', '3'] + ]) + expect(headers.get('a')).to.equal('1, 3') + expect(headers.get('b')).to.equal('2') + + headers = new Headers([ + new Set(['a', '1']), + ['b', '2'], + new Map([['a', null], ['3', null]]).keys() + ]) + expect(headers.get('a')).to.equal('1, 3') + expect(headers.get('b')).to.equal('2') + + headers = new Headers(new Map([ + ['a', '1'], + ['b', '2'] + ])) + expect(headers.get('a')).to.equal('1') + expect(headers.get('b')).to.equal('2') + }) + + it('should throw a TypeError if non-tuple exists in a headers initializer', () => { + expect(() => new Headers([['b', '2', 'huh?']])).to.throw(TypeError) + expect(() => new Headers(['b2'])).to.throw(TypeError) + expect(() => new Headers('b2')).to.throw(TypeError) + // expect(() => new Headers({ [Symbol.iterator]: 42 })).to.throw(TypeError) + }) + + xit('should use a custom inspect function', () => { + const headers = new Headers([ + ['Host', 'thehost'], + ['Host', 'notthehost'], + ['a', '1'], + ['b', '2'], + ['a', '3'] + ]) + + // eslint-disable-next-line quotes + expect(format(headers)).to.equal("{ a: [ '1', '3' ], b: '2', host: 'thehost' }") + }) +}) diff --git a/test/node-fetch/main.js b/test/node-fetch/main.js new file mode 100644 index 00000000000..62707936def --- /dev/null +++ b/test/node-fetch/main.js @@ -0,0 +1,1662 @@ +/* eslint no-unused-expressions: "off" */ +/* globals AbortController */ + +// Test tools +const zlib = require('zlib') +const stream = require('stream') +const vm = require('vm') +const chai = require('chai') +const crypto = require('crypto') +const chaiPromised = require('chai-as-promised') +const chaiIterator = require('chai-iterator') +const chaiString = require('chai-string') +const delay = require('delay') +const { Blob } = require('buffer') + +const { + fetch, + Headers, + Request, + Response, + setGlobalDispatcher, + Agent +} = require('../../index.js') +const HeadersOrig = require('../../lib/api/api-fetch/headers.js').Headers +const RequestOrig = require('../../lib/api/api-fetch/request.js').Request +const ResponseOrig = require('../../lib/api/api-fetch/response.js').Response +const TestServer = require('./utils/server.js') +const chaiTimeout = require('./utils/chai-timeout.js') +const toWeb = require('./utils/toWeb.js') +const { ReadableStream } = require('stream/web') + +function isNodeLowerThan (version) { + return !~process.version.localeCompare(version, undefined, { numeric: true }) +} + +const { + Uint8Array: VMUint8Array +} = vm.runInNewContext('this') + +chai.use(chaiPromised) +chai.use(chaiIterator) +chai.use(chaiString) +chai.use(chaiTimeout) +const { expect } = chai + +describe('node-fetch', () => { + const local = new TestServer() + let base + + before(async () => { + await local.start() + setGlobalDispatcher(new Agent({ + connect: { + rejectUnauthorized: false + } + })) + base = `http://${local.hostname}:${local.port}/` + }) + + after(async () => { + return local.stop() + }) + + it('should return a promise', () => { + const url = `${base}hello` + const p = fetch(url) + expect(p).to.be.an.instanceof(Promise) + expect(p).to.have.property('then') + }) + + it('should expose Headers, Response and Request constructors', () => { + expect(Headers).to.equal(HeadersOrig) + expect(Response).to.equal(ResponseOrig) + expect(Request).to.equal(RequestOrig) + }) + + it('should support proper toString output for Headers, Response and Request objects', () => { + expect(new Headers().toString()).to.equal('[object Headers]') + expect(new Response().toString()).to.equal('[object Response]') + expect(new Request(base).toString()).to.equal('[object Request]') + }) + + it('should reject with error if url is protocol relative', () => { + const url = '//example.com/' + return expect(fetch(url)).to.eventually.be.rejectedWith(TypeError, /Invalid URL/) + }) + + it('should reject with error if url is relative path', () => { + const url = '/some/path' + return expect(fetch(url)).to.eventually.be.rejectedWith(TypeError, /Invalid URL/) + }) + + it('should reject with error if protocol is unsupported', () => { + const url = 'ftp://example.com/' + return expect(fetch(url)).to.eventually.be.rejectedWith(TypeError) + }) + + it('should reject with error on network failure', function () { + this.timeout(5000) + const url = 'http://localhost:50000/' + return expect(fetch(url)).to.eventually.be.rejected + .and.be.an.instanceOf(TypeError) + }) + + it('should resolve into response', () => { + const url = `${base}hello` + return fetch(url).then(res => { + expect(res).to.be.an.instanceof(Response) + expect(res.headers).to.be.an.instanceof(Headers) + expect(res.body).to.be.an.instanceof(ReadableStream) + expect(res.bodyUsed).to.be.false + + expect(res.url).to.equal(url) + expect(res.ok).to.be.true + expect(res.status).to.equal(200) + expect(res.statusText).to.equal('OK') + }) + }) + + it('Response.redirect should resolve into response', () => { + const res = Response.redirect('http://localhost') + expect(res).to.be.an.instanceof(Response) + expect(res.headers).to.be.an.instanceof(Headers) + expect(res.headers.get('location')).to.equal('http://localhost/') + expect(res.status).to.equal(302) + }) + + it('Response.redirect /w invalid url should fail', () => { + expect(() => { + Response.redirect('localhost') + }).to.throw() + }) + + it('Response.redirect /w invalid status should fail', () => { + expect(() => { + Response.redirect('http://localhost', 200) + }).to.throw() + }) + + it('should accept plain text response', () => { + const url = `${base}plain` + return fetch(url).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain') + return res.text().then(result => { + expect(res.bodyUsed).to.be.true + expect(result).to.be.a('string') + expect(result).to.equal('text') + }) + }) + }) + + it('should accept html response (like plain text)', () => { + const url = `${base}html` + return fetch(url).then(res => { + expect(res.headers.get('content-type')).to.equal('text/html') + return res.text().then(result => { + expect(res.bodyUsed).to.be.true + expect(result).to.be.a('string') + expect(result).to.equal('') + }) + }) + }) + + it('should accept json response', () => { + const url = `${base}json` + return fetch(url).then(res => { + expect(res.headers.get('content-type')).to.equal('application/json') + return res.json().then(result => { + expect(res.bodyUsed).to.be.true + expect(result).to.be.an('object') + expect(result).to.deep.equal({ name: 'value' }) + }) + }) + }) + + it('should send request with custom headers', () => { + const url = `${base}inspect` + const options = { + headers: { 'x-custom-header': 'abc' } + } + return fetch(url, options).then(res => { + return res.json() + }).then(res => { + expect(res.headers['x-custom-header']).to.equal('abc') + }) + }) + + it('should accept headers instance', () => { + const url = `${base}inspect` + const options = { + headers: new Headers({ 'x-custom-header': 'abc' }) + } + return fetch(url, options).then(res => { + return res.json() + }).then(res => { + expect(res.headers['x-custom-header']).to.equal('abc') + }) + }) + + it('should follow redirect code 301', () => { + const url = `${base}redirect/301` + return fetch(url).then(res => { + expect(res.url).to.equal(`${base}inspect`) + expect(res.status).to.equal(200) + expect(res.ok).to.be.true + }) + }) + + it('should follow redirect code 302', () => { + const url = `${base}redirect/302` + return fetch(url).then(res => { + expect(res.url).to.equal(`${base}inspect`) + expect(res.status).to.equal(200) + }) + }) + + it('should follow redirect code 303', () => { + const url = `${base}redirect/303` + return fetch(url).then(res => { + expect(res.url).to.equal(`${base}inspect`) + expect(res.status).to.equal(200) + }) + }) + + it('should follow redirect code 307', () => { + const url = `${base}redirect/307` + return fetch(url).then(res => { + expect(res.url).to.equal(`${base}inspect`) + expect(res.status).to.equal(200) + }) + }) + + it('should follow redirect code 308', () => { + const url = `${base}redirect/308` + return fetch(url).then(res => { + expect(res.url).to.equal(`${base}inspect`) + expect(res.status).to.equal(200) + }) + }) + + it('should follow redirect chain', () => { + const url = `${base}redirect/chain` + return fetch(url).then(res => { + expect(res.url).to.equal(`${base}inspect`) + expect(res.status).to.equal(200) + }) + }) + + it('should follow POST request redirect code 301 with GET', () => { + const url = `${base}redirect/301` + const options = { + method: 'POST', + body: 'a=1' + } + return fetch(url, options).then(res => { + expect(res.url).to.equal(`${base}inspect`) + expect(res.status).to.equal(200) + return res.json().then(result => { + expect(result.method).to.equal('GET') + expect(result.body).to.equal('') + }) + }) + }) + + it('should follow PATCH request redirect code 301 with PATCH', () => { + const url = `${base}redirect/301` + const options = { + method: 'PATCH', + body: 'a=1' + } + return fetch(url, options).then(res => { + expect(res.url).to.equal(`${base}inspect`) + expect(res.status).to.equal(200) + return res.json().then(res => { + expect(res.method).to.equal('PATCH') + expect(res.body).to.equal('a=1') + }) + }) + }) + + it('should follow POST request redirect code 302 with GET', () => { + const url = `${base}redirect/302` + const options = { + method: 'POST', + body: 'a=1' + } + return fetch(url, options).then(res => { + expect(res.url).to.equal(`${base}inspect`) + expect(res.status).to.equal(200) + return res.json().then(result => { + expect(result.method).to.equal('GET') + expect(result.body).to.equal('') + }) + }) + }) + + it('should follow PATCH request redirect code 302 with PATCH', () => { + const url = `${base}redirect/302` + const options = { + method: 'PATCH', + body: 'a=1' + } + return fetch(url, options).then(res => { + expect(res.url).to.equal(`${base}inspect`) + expect(res.status).to.equal(200) + return res.json().then(res => { + expect(res.method).to.equal('PATCH') + expect(res.body).to.equal('a=1') + }) + }) + }) + + it('should follow redirect code 303 with GET', () => { + const url = `${base}redirect/303` + const options = { + method: 'PUT', + body: 'a=1' + } + return fetch(url, options).then(res => { + expect(res.url).to.equal(`${base}inspect`) + expect(res.status).to.equal(200) + return res.json().then(result => { + expect(result.method).to.equal('GET') + expect(result.body).to.equal('') + }) + }) + }) + + it('should follow PATCH request redirect code 307 with PATCH', () => { + const url = `${base}redirect/307` + const options = { + method: 'PATCH', + body: 'a=1' + } + return fetch(url, options).then(res => { + expect(res.url).to.equal(`${base}inspect`) + expect(res.status).to.equal(200) + return res.json().then(result => { + expect(result.method).to.equal('PATCH') + expect(result.body).to.equal('a=1') + }) + }) + }) + + it('should not follow non-GET redirect if body is a readable stream', () => { + const url = `${base}redirect/307` + const options = { + method: 'PATCH', + body: toWeb(stream.Readable.from('tada')) + } + return expect(fetch(url, options)).to.eventually.be.rejected + .and.be.an.instanceOf(TypeError) + }) + + it('should obey maximum redirect, reject case', () => { + const url = `${base}redirect/chain/20` + return expect(fetch(url)).to.eventually.be.rejected + .and.be.an.instanceOf(TypeError) + }) + + it('should obey redirect chain, resolve case', () => { + const url = `${base}redirect/chain/19` + return fetch(url).then(res => { + expect(res.url).to.equal(`${base}inspect`) + expect(res.status).to.equal(200) + }) + }) + + it('should support redirect mode, error flag', () => { + const url = `${base}redirect/301` + const options = { + redirect: 'error' + } + return expect(fetch(url, options)).to.eventually.be.rejected + .and.be.an.instanceOf(TypeError) + }) + + it('should support redirect mode, manual flag when there is no redirect', () => { + const url = `${base}hello` + const options = { + redirect: 'manual' + } + return fetch(url, options).then(res => { + expect(res.url).to.equal(url) + expect(res.status).to.equal(200) + expect(res.headers.get('location')).to.be.null + }) + }) + + it('should follow redirect code 301 and keep existing headers', () => { + const url = `${base}redirect/301` + const options = { + headers: new Headers({ 'x-custom-header': 'abc' }) + } + return fetch(url, options).then(res => { + expect(res.url).to.equal(`${base}inspect`) + return res.json() + }).then(res => { + expect(res.headers['x-custom-header']).to.equal('abc') + }) + }) + + it('should treat broken redirect as ordinary response (follow)', () => { + const url = `${base}redirect/no-location` + return fetch(url).then(res => { + expect(res.url).to.equal(url) + expect(res.status).to.equal(301) + expect(res.headers.get('location')).to.be.null + }) + }) + + xit('should treat broken redirect as ordinary response (manual)', () => { + const url = `${base}redirect/no-location` + const options = { + redirect: 'manual' + } + return fetch(url, options).then(res => { + expect(res.url).to.equal(url) + expect(res.status).to.equal(301) + expect(res.headers.get('location')).to.be.null + }) + }) + + it('should throw a TypeError on an invalid redirect option', () => { + const url = `${base}redirect/301` + const options = { + redirect: 'foobar' + } + return fetch(url, options).then(() => { + expect.fail() + }, error => { + expect(error).to.be.an.instanceOf(TypeError) + }) + }) + + it('should set redirected property on response when redirect', () => { + const url = `${base}redirect/301` + return fetch(url).then(res => { + expect(res.redirected).to.be.true + }) + }) + + it('should not set redirected property on response without redirect', () => { + const url = `${base}hello` + return fetch(url).then(res => { + expect(res.redirected).to.be.false + }) + }) + + it('should handle client-error response', () => { + const url = `${base}error/400` + return fetch(url).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain') + expect(res.status).to.equal(400) + expect(res.statusText).to.equal('Bad Request') + expect(res.ok).to.be.false + return res.text().then(result => { + expect(res.bodyUsed).to.be.true + expect(result).to.be.a('string') + expect(result).to.equal('client error') + }) + }) + }) + + it('should handle server-error response', () => { + const url = `${base}error/500` + return fetch(url).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain') + expect(res.status).to.equal(500) + expect(res.statusText).to.equal('Internal Server Error') + expect(res.ok).to.be.false + return res.text().then(result => { + expect(res.bodyUsed).to.be.true + expect(result).to.be.a('string') + expect(result).to.equal('server error') + }) + }) + }) + + it('should handle network-error response', () => { + const url = `${base}error/reset` + return expect(fetch(url)).to.eventually.be.rejectedWith(TypeError) + }) + + it('should handle network-error partial response', () => { + const url = `${base}error/premature` + return fetch(url).then(res => { + expect(res.status).to.equal(200) + expect(res.ok).to.be.true + return expect(res.text()).to.eventually.be.rejectedWith(Error) + }) + }) + + it('should handle network-error in chunked response async iterator', () => { + const url = `${base}error/premature/chunked` + return fetch(url).then(res => { + expect(res.status).to.equal(200) + expect(res.ok).to.be.true + + const read = async body => { + const chunks = [] + for await (const chunk of body) { + chunks.push(chunk) + } + + return chunks + } + + return expect(read(res.body)) + .to.eventually.be.rejectedWith(Error) + }) + }) + + it('should handle network-error in chunked response in consumeBody', () => { + const url = `${base}error/premature/chunked` + return fetch(url).then(res => { + expect(res.status).to.equal(200) + expect(res.ok).to.be.true + + return expect(res.text()).to.eventually.be.rejectedWith(Error) + }) + }) + + it('should handle DNS-error response', () => { + const url = 'http://domain.invalid' + return expect(fetch(url)).to.eventually.be.rejectedWith(TypeError) + }) + + it('should reject invalid json response', () => { + const url = `${base}error/json` + return fetch(url).then(res => { + expect(res.headers.get('content-type')).to.equal('application/json') + return expect(res.json()).to.eventually.be.rejectedWith(Error) + }) + }) + + xit('should handle response with no status text', () => { + const url = `${base}no-status-text` + return fetch(url).then(res => { + expect(res.statusText).to.equal('') + }) + }) + + it('should handle no content response', () => { + const url = `${base}no-content` + return fetch(url).then(res => { + expect(res.status).to.equal(204) + expect(res.statusText).to.equal('No Content') + expect(res.ok).to.be.true + return res.text().then(result => { + expect(result).to.be.a('string') + expect(result).to.be.empty + }) + }) + }) + + it('should reject when trying to parse no content response as json', () => { + const url = `${base}no-content` + return fetch(url).then(res => { + expect(res.status).to.equal(204) + expect(res.statusText).to.equal('No Content') + expect(res.ok).to.be.true + return expect(res.json()).to.eventually.be.rejectedWith(Error) + }) + }) + + xit('should handle no content response with gzip encoding', () => { + const url = `${base}no-content/gzip` + return fetch(url).then(res => { + expect(res.status).to.equal(204) + expect(res.statusText).to.equal('No Content') + expect(res.headers.get('content-encoding')).to.equal('gzip') + expect(res.ok).to.be.true + return res.text().then(result => { + expect(result).to.be.a('string') + expect(result).to.be.empty + }) + }) + }) + + xit('should handle not modified response', () => { + const url = `${base}not-modified` + return fetch(url).then(res => { + expect(res.status).to.equal(304) + expect(res.statusText).to.equal('Not Modified') + expect(res.ok).to.be.false + return res.text().then(result => { + expect(result).to.be.a('string') + expect(result).to.be.empty + }) + }) + }) + + xit('should handle not modified response with gzip encoding', () => { + const url = `${base}not-modified/gzip` + return fetch(url).then(res => { + expect(res.status).to.equal(304) + expect(res.statusText).to.equal('Not Modified') + expect(res.headers.get('content-encoding')).to.equal('gzip') + expect(res.ok).to.be.false + return res.text().then(result => { + expect(result).to.be.a('string') + expect(result).to.be.empty + }) + }) + }) + + xit('should decompress gzip response', () => { + const url = `${base}gzip` + return fetch(url).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain') + return res.text().then(result => { + expect(result).to.be.a('string') + expect(result).to.equal('hello world') + }) + }) + }) + + xit('should decompress slightly invalid gzip response', () => { + const url = `${base}gzip-truncated` + return fetch(url).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain') + return res.text().then(result => { + expect(result).to.be.a('string') + expect(result).to.equal('hello world') + }) + }) + }) + + xit('should make capitalised Content-Encoding lowercase', () => { + const url = `${base}gzip-capital` + return fetch(url).then(res => { + expect(res.headers.get('content-encoding')).to.equal('gzip') + return res.text().then(result => { + expect(result).to.be.a('string') + expect(result).to.equal('hello world') + }) + }) + }) + + xit('should decompress deflate response', () => { + const url = `${base}deflate` + return fetch(url).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain') + return res.text().then(result => { + expect(result).to.be.a('string') + expect(result).to.equal('hello world') + }) + }) + }) + + xit('should decompress deflate raw response from old apache server', () => { + const url = `${base}deflate-raw` + return fetch(url).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain') + return res.text().then(result => { + expect(result).to.be.a('string') + expect(result).to.equal('hello world') + }) + }) + }) + + xit('should decompress brotli response', function () { + if (typeof zlib.createBrotliDecompress !== 'function') { + this.skip() + } + + const url = `${base}brotli` + return fetch(url).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain') + return res.text().then(result => { + expect(result).to.be.a('string') + expect(result).to.equal('hello world') + }) + }) + }) + + xit('should handle no content response with brotli encoding', function () { + if (typeof zlib.createBrotliDecompress !== 'function') { + this.skip() + } + + const url = `${base}no-content/brotli` + return fetch(url).then(res => { + expect(res.status).to.equal(204) + expect(res.statusText).to.equal('No Content') + expect(res.headers.get('content-encoding')).to.equal('br') + expect(res.ok).to.be.true + return res.text().then(result => { + expect(result).to.be.a('string') + expect(result).to.be.empty + }) + }) + }) + + xit('should skip decompression if unsupported', () => { + const url = `${base}sdch` + return fetch(url).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain') + return res.text().then(result => { + expect(result).to.be.a('string') + expect(result).to.equal('fake sdch string') + }) + }) + }) + + xit('should reject if response compression is invalid', () => { + const url = `${base}invalid-content-encoding` + return fetch(url).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain') + return expect(res.text()).to.eventually.be.rejected + .and.be.an.instanceOf(TypeError) + .and.have.property('code', 'Z_DATA_ERROR') + }) + }) + + it('should handle errors on the body stream even if it is not used', done => { + const url = `${base}invalid-content-encoding` + fetch(url) + .then(res => { + expect(res.status).to.equal(200) + }) + .catch(() => {}) + .then(() => { + // Wait a few ms to see if a uncaught error occurs + setTimeout(() => { + done() + }, 20) + }) + }) + + xit('should collect handled errors on the body stream to reject if the body is used later', () => { + const url = `${base}invalid-content-encoding` + return fetch(url).then(delay(20)).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain') + return expect(res.text()).to.eventually.be.rejected + .and.be.an.instanceOf(TypeError) + .and.have.property('code', 'Z_DATA_ERROR') + }) + }) + + xit('should allow disabling auto decompression', () => { + const url = `${base}gzip` + const options = { + compress: false + } + return fetch(url, options).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain') + return res.text().then(result => { + expect(result).to.be.a('string') + expect(result).to.not.equal('hello world') + }) + }) + }) + + xit('should not overwrite existing accept-encoding header when auto decompression is true', () => { + const url = `${base}inspect` + const options = { + compress: true, + headers: { + 'Accept-Encoding': 'gzip' + } + } + return fetch(url, options).then(res => res.json()).then(res => { + expect(res.headers['accept-encoding']).to.equal('gzip') + }) + }) + + const testAbortController = (name, buildAbortController, moreTests = null) => { + describe(`AbortController (${name})`, () => { + let controller + + beforeEach(() => { + controller = buildAbortController() + }) + + it('should support request cancellation with signal', () => { + const fetches = [ + fetch( + `${base}timeout`, + { + method: 'POST', + signal: controller.signal, + headers: { + 'Content-Type': 'application/json', + body: JSON.stringify({ hello: 'world' }) + } + } + ) + ] + setTimeout(() => { + controller.abort() + }, 100) + + return Promise.all(fetches.map(fetched => expect(fetched) + .to.eventually.be.rejected + .and.be.an.instanceOf(Error) + .and.have.property('name', 'AbortError') + )) + }) + + it('should support multiple request cancellation with signal', () => { + const fetches = [ + fetch(`${base}timeout`, { signal: controller.signal }), + fetch( + `${base}timeout`, + { + method: 'POST', + signal: controller.signal, + headers: { + 'Content-Type': 'application/json', + body: JSON.stringify({ hello: 'world' }) + } + } + ) + ] + setTimeout(() => { + controller.abort() + }, 100) + + return Promise.all(fetches.map(fetched => expect(fetched) + .to.eventually.be.rejected + .and.be.an.instanceOf(Error) + .and.have.property('name', 'AbortError') + )) + }) + + it('should reject immediately if signal has already been aborted', () => { + const url = `${base}timeout` + const options = { + signal: controller.signal + } + controller.abort() + const fetched = fetch(url, options) + return expect(fetched).to.eventually.be.rejected + .and.be.an.instanceOf(Error) + .and.have.property('name', 'AbortError') + }) + + it('should allow redirects to be aborted', () => { + const request = new Request(`${base}redirect/slow`, { + signal: controller.signal + }) + setTimeout(() => { + controller.abort() + }, 20) + return expect(fetch(request)).to.be.eventually.rejected + .and.be.an.instanceOf(Error) + .and.have.property('name', 'AbortError') + }) + + it('should allow redirected response body to be aborted', () => { + const request = new Request(`${base}redirect/slow-stream`, { + signal: controller.signal + }) + return expect(fetch(request).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain') + const result = res.text() + controller.abort() + return result + })).to.be.eventually.rejected + .and.be.an.instanceOf(Error) + .and.have.property('name', 'AbortError') + }) + + it('should reject response body with AbortError when aborted before stream has been read completely', () => { + return expect(fetch( + `${base}slow`, + { signal: controller.signal } + )) + .to.eventually.be.fulfilled + .then(res => { + const promise = res.text() + controller.abort() + return expect(promise) + .to.eventually.be.rejected + .and.be.an.instanceof(Error) + .and.have.property('name', 'AbortError') + }) + }) + + it('should reject response body methods immediately with AbortError when aborted before stream is disturbed', () => { + return expect(fetch( + `${base}slow`, + { signal: controller.signal } + )) + .to.eventually.be.fulfilled + .then(res => { + controller.abort() + return expect(res.text()) + .to.eventually.be.rejected + .and.be.an.instanceof(Error) + .and.have.property('name', 'AbortError') + }) + }) + + xit('should cancel request body of type Stream with AbortError when aborted', () => { + const body = toWeb(new stream.Readable({ objectMode: true })) + body._read = () => {} + const promise = fetch( + `${base}slow`, + { signal: controller.signal, body, method: 'POST' } + ) + + const result = Promise.all([ + new Promise((resolve, reject) => { + body.on('error', error => { + try { + expect(error).to.be.an.instanceof(Error) + .and.have.property('name', 'AbortError') + resolve() + } catch (error_) { + reject(error_) + } + }) + }), + expect(promise).to.eventually.be.rejected + .and.be.an.instanceof(Error) + .and.have.property('name', 'AbortError') + ]) + + controller.abort() + + return result + }) + + if (moreTests) { + moreTests() + } + }) + } + + testAbortController('native', () => new AbortController()) + + it('should throw a TypeError if a signal is not of type AbortSignal or EventTarget', () => { + return Promise.all([ + expect(fetch(`${base}inspect`, { signal: {} })) + .to.be.eventually.rejected + .and.be.an.instanceof(TypeError), + expect(fetch(`${base}inspect`, { signal: '' })) + .to.be.eventually.rejected + .and.be.an.instanceof(TypeError), + expect(fetch(`${base}inspect`, { signal: Object.create(null) })) + .to.be.eventually.rejected + .and.be.an.instanceof(TypeError) + ]) + }) + + it('should gracefully handle a nullish signal', () => { + return Promise.all([ + fetch(`${base}hello`, { signal: null }).then(res => { + return expect(res.ok).to.be.true + }), + fetch(`${base}hello`, { signal: undefined }).then(res => { + return expect(res.ok).to.be.true + }) + ]) + }) + + it('should allow setting User-Agent', () => { + const url = `${base}inspect` + const options = { + headers: { + 'user-agent': 'faked' + } + } + return fetch(url, options).then(res => res.json()).then(res => { + expect(res.headers['user-agent']).to.equal('faked') + }) + }) + + it('should set default Accept header', () => { + const url = `${base}inspect` + fetch(url).then(res => res.json()).then(res => { + expect(res.headers.accept).to.equal('*/*') + }) + }) + + it('should allow setting Accept header', () => { + const url = `${base}inspect` + const options = { + headers: { + accept: 'application/json' + } + } + return fetch(url, options).then(res => res.json()).then(res => { + expect(res.headers.accept).to.equal('application/json') + }) + }) + + it('should allow POST request', () => { + const url = `${base}inspect` + const options = { + method: 'POST' + } + return fetch(url, options).then(res => { + return res.json() + }).then(res => { + expect(res.method).to.equal('POST') + expect(res.headers['transfer-encoding']).to.be.undefined + expect(res.headers['content-type']).to.be.undefined + expect(res.headers['content-length']).to.equal('0') + }) + }) + + it('should allow POST request with string body', () => { + const url = `${base}inspect` + const options = { + method: 'POST', + body: 'a=1' + } + return fetch(url, options).then(res => { + return res.json() + }).then(res => { + expect(res.method).to.equal('POST') + expect(res.body).to.equal('a=1') + expect(res.headers['transfer-encoding']).to.be.undefined + expect(res.headers['content-type']).to.equal('text/plain;charset=UTF-8') + expect(res.headers['content-length']).to.equal('3') + }) + }) + + it('should allow POST request with buffer body', () => { + const url = `${base}inspect` + const options = { + method: 'POST', + body: Buffer.from('a=1', 'utf-8') + } + return fetch(url, options).then(res => { + return res.json() + }).then(res => { + expect(res.method).to.equal('POST') + expect(res.body).to.equal('a=1') + expect(res.headers['transfer-encoding']).to.be.undefined + expect(res.headers['content-type']).to.be.undefined + expect(res.headers['content-length']).to.equal('3') + }) + }) + + it('should allow POST request with ArrayBuffer body', () => { + const encoder = new TextEncoder() + const url = `${base}inspect` + const options = { + method: 'POST', + body: encoder.encode('Hello, world!\n').buffer + } + return fetch(url, options).then(res => res.json()).then(res => { + expect(res.method).to.equal('POST') + expect(res.body).to.equal('Hello, world!\n') + expect(res.headers['transfer-encoding']).to.be.undefined + expect(res.headers['content-type']).to.be.undefined + expect(res.headers['content-length']).to.equal('14') + }) + }) + + it('should allow POST request with ArrayBuffer body from a VM context', () => { + const url = `${base}inspect` + const options = { + method: 'POST', + body: new VMUint8Array(Buffer.from('Hello, world!\n')).buffer + } + return fetch(url, options).then(res => res.json()).then(res => { + expect(res.method).to.equal('POST') + expect(res.body).to.equal('Hello, world!\n') + expect(res.headers['transfer-encoding']).to.be.undefined + expect(res.headers['content-type']).to.be.undefined + expect(res.headers['content-length']).to.equal('14') + }) + }) + + it('should allow POST request with ArrayBufferView (Uint8Array) body', () => { + const encoder = new TextEncoder() + const url = `${base}inspect` + const options = { + method: 'POST', + body: encoder.encode('Hello, world!\n') + } + return fetch(url, options).then(res => res.json()).then(res => { + expect(res.method).to.equal('POST') + expect(res.body).to.equal('Hello, world!\n') + expect(res.headers['transfer-encoding']).to.be.undefined + expect(res.headers['content-type']).to.be.undefined + expect(res.headers['content-length']).to.equal('14') + }) + }) + + it('should allow POST request with ArrayBufferView (DataView) body', () => { + const encoder = new TextEncoder() + const url = `${base}inspect` + const options = { + method: 'POST', + body: new DataView(encoder.encode('Hello, world!\n').buffer) + } + return fetch(url, options).then(res => res.json()).then(res => { + expect(res.method).to.equal('POST') + expect(res.body).to.equal('Hello, world!\n') + expect(res.headers['transfer-encoding']).to.be.undefined + expect(res.headers['content-type']).to.be.undefined + expect(res.headers['content-length']).to.equal('14') + }) + }) + + it('should allow POST request with ArrayBufferView (Uint8Array) body from a VM context', () => { + const url = `${base}inspect` + const options = { + method: 'POST', + body: new VMUint8Array(Buffer.from('Hello, world!\n')) + } + return fetch(url, options).then(res => res.json()).then(res => { + expect(res.method).to.equal('POST') + expect(res.body).to.equal('Hello, world!\n') + expect(res.headers['transfer-encoding']).to.be.undefined + expect(res.headers['content-type']).to.be.undefined + expect(res.headers['content-length']).to.equal('14') + }) + }) + + it('should allow POST request with ArrayBufferView (Uint8Array, offset, length) body', () => { + const encoder = new TextEncoder() + const url = `${base}inspect` + const options = { + method: 'POST', + body: encoder.encode('Hello, world!\n').subarray(7, 13) + } + return fetch(url, options).then(res => res.json()).then(res => { + expect(res.method).to.equal('POST') + expect(res.body).to.equal('world!') + expect(res.headers['transfer-encoding']).to.be.undefined + expect(res.headers['content-type']).to.be.undefined + expect(res.headers['content-length']).to.equal('6') + }) + }) + + it('should allow POST request with blob body without type', () => { + const url = `${base}inspect` + const options = { + method: 'POST', + body: new Blob(['a=1']) + } + return fetch(url, options).then(res => { + return res.json() + }).then(res => { + expect(res.method).to.equal('POST') + expect(res.body).to.equal('a=1') + expect(res.headers['transfer-encoding']).to.be.undefined + // expect(res.headers['content-type']).to.be.undefined + expect(res.headers['content-length']).to.equal('3') + }) + }) + + it('should allow POST request with blob body with type', () => { + const url = `${base}inspect` + const options = { + method: 'POST', + body: new Blob(['a=1'], { + type: 'text/plain;charset=UTF-8' + }) + } + return fetch(url, options).then(res => { + return res.json() + }).then(res => { + expect(res.method).to.equal('POST') + expect(res.body).to.equal('a=1') + expect(res.headers['transfer-encoding']).to.be.undefined + expect(res.headers['content-type']).to.equal('text/plain;charset=utf-8') + expect(res.headers['content-length']).to.equal('3') + }) + }) + + it('should allow POST request with readable stream as body', () => { + const url = `${base}inspect` + const options = { + method: 'POST', + body: toWeb(stream.Readable.from('a=1')) + } + return fetch(url, options).then(res => { + return res.json() + }).then(res => { + expect(res.method).to.equal('POST') + expect(res.body).to.equal('a=1') + expect(res.headers['transfer-encoding']).to.equal('chunked') + expect(res.headers['content-type']).to.be.undefined + expect(res.headers['content-length']).to.be.undefined + }) + }) + + it('should allow POST request with object body', () => { + const url = `${base}inspect` + // Note that fetch simply calls tostring on an object + const options = { + method: 'POST', + body: { a: 1 } + } + return fetch(url, options).then(res => { + return res.json() + }).then(res => { + expect(res.method).to.equal('POST') + expect(res.body).to.equal('[object Object]') + expect(res.headers['content-type']).to.equal('text/plain;charset=UTF-8') + expect(res.headers['content-length']).to.equal('15') + }) + }) + + it('constructing a Response with URLSearchParams as body should have a Content-Type', () => { + const parameters = new URLSearchParams() + const res = new Response(parameters) + res.headers.get('Content-Type') + expect(res.headers.get('Content-Type')).to.equal('application/x-www-form-urlencoded;charset=UTF-8') + }) + + it('constructing a Request with URLSearchParams as body should have a Content-Type', () => { + const parameters = new URLSearchParams() + const request = new Request(base, { method: 'POST', body: parameters }) + expect(request.headers.get('Content-Type')).to.equal('application/x-www-form-urlencoded;charset=UTF-8') + }) + + it('Reading a body with URLSearchParams should echo back the result', () => { + const parameters = new URLSearchParams() + parameters.append('a', '1') + return new Response(parameters).text().then(text => { + expect(text).to.equal('a=1') + }) + }) + + // Body should been cloned... + it('constructing a Request/Response with URLSearchParams and mutating it should not affected body', () => { + const parameters = new URLSearchParams() + const request = new Request(`${base}inspect`, { method: 'POST', body: parameters }) + parameters.append('a', '1') + return request.text().then(text => { + expect(text).to.equal('') + }) + }) + + it('should allow POST request with URLSearchParams as body', () => { + const parameters = new URLSearchParams() + parameters.append('a', '1') + + const url = `${base}inspect` + const options = { + method: 'POST', + body: parameters + } + return fetch(url, options).then(res => { + return res.json() + }).then(res => { + expect(res.method).to.equal('POST') + expect(res.headers['content-type']).to.equal('application/x-www-form-urlencoded;charset=UTF-8') + expect(res.headers['content-length']).to.equal('3') + expect(res.body).to.equal('a=1') + }) + }) + + it('should still recognize URLSearchParams when extended', () => { + class CustomSearchParameters extends URLSearchParams {} + const parameters = new CustomSearchParameters() + parameters.append('a', '1') + + const url = `${base}inspect` + const options = { + method: 'POST', + body: parameters + } + return fetch(url, options).then(res => { + return res.json() + }).then(res => { + expect(res.method).to.equal('POST') + expect(res.headers['content-type']).to.equal('application/x-www-form-urlencoded;charset=UTF-8') + expect(res.headers['content-length']).to.equal('3') + expect(res.body).to.equal('a=1') + }) + }) + + it('should allow PUT request', () => { + const url = `${base}inspect` + const options = { + method: 'PUT', + body: 'a=1' + } + return fetch(url, options).then(res => { + return res.json() + }).then(res => { + expect(res.method).to.equal('PUT') + expect(res.body).to.equal('a=1') + }) + }) + + it('should allow DELETE request', () => { + const url = `${base}inspect` + const options = { + method: 'DELETE' + } + return fetch(url, options).then(res => { + return res.json() + }).then(res => { + expect(res.method).to.equal('DELETE') + }) + }) + + it('should allow DELETE request with string body', () => { + const url = `${base}inspect` + const options = { + method: 'DELETE', + body: 'a=1' + } + return fetch(url, options).then(res => { + return res.json() + }).then(res => { + expect(res.method).to.equal('DELETE') + expect(res.body).to.equal('a=1') + expect(res.headers['transfer-encoding']).to.be.undefined + expect(res.headers['content-length']).to.equal('3') + }) + }) + + it('should allow PATCH request', () => { + const url = `${base}inspect` + const options = { + method: 'PATCH', + body: 'a=1' + } + return fetch(url, options).then(res => { + return res.json() + }).then(res => { + expect(res.method).to.equal('PATCH') + expect(res.body).to.equal('a=1') + }) + }) + + it('should allow HEAD request', () => { + const url = `${base}hello` + const options = { + method: 'HEAD' + } + return fetch(url, options).then(res => { + expect(res.status).to.equal(200) + expect(res.statusText).to.equal('OK') + expect(res.headers.get('content-type')).to.equal('text/plain') + // expect(res.body).to.be.an.instanceof(stream.Transform) + return res.text() + }).then(text => { + expect(text).to.equal('') + }) + }) + + it('should allow HEAD request with content-encoding header', () => { + const url = `${base}error/404` + const options = { + method: 'HEAD' + } + return fetch(url, options).then(res => { + expect(res.status).to.equal(404) + expect(res.headers.get('content-encoding')).to.equal('gzip') + return res.text() + }).then(text => { + expect(text).to.equal('') + }) + }) + + it('should allow OPTIONS request', () => { + const url = `${base}options` + const options = { + method: 'OPTIONS' + } + return fetch(url, options).then(res => { + expect(res.status).to.equal(200) + expect(res.statusText).to.equal('OK') + expect(res.headers.get('allow')).to.equal('GET, HEAD, OPTIONS') + // expect(res.body).to.be.an.instanceof(stream.Transform) + }) + }) + + it('should reject decoding body twice', () => { + const url = `${base}plain` + return fetch(url).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain') + return res.text().then(() => { + expect(res.bodyUsed).to.be.true + return expect(res.text()).to.eventually.be.rejectedWith(Error) + }) + }) + }) + + it('should allow cloning a json response and log it as text response', () => { + const url = `${base}json` + return fetch(url).then(res => { + const r1 = res.clone() + return Promise.all([res.json(), r1.text()]).then(results => { + expect(results[0]).to.deep.equal({ name: 'value' }) + expect(results[1]).to.equal('{"name":"value"}') + }) + }) + }) + + it('should allow cloning a json response, and then log it as text response', () => { + const url = `${base}json` + return fetch(url).then(res => { + const r1 = res.clone() + return res.json().then(result => { + expect(result).to.deep.equal({ name: 'value' }) + return r1.text().then(result => { + expect(result).to.equal('{"name":"value"}') + }) + }) + }) + }) + + it('should allow cloning a json response, first log as text response, then return json object', () => { + const url = `${base}json` + return fetch(url).then(res => { + const r1 = res.clone() + return r1.text().then(result => { + expect(result).to.equal('{"name":"value"}') + return res.json().then(result => { + expect(result).to.deep.equal({ name: 'value' }) + }) + }) + }) + }) + + it('should not allow cloning a response after its been used', () => { + const url = `${base}hello` + return fetch(url).then(res => + res.text().then(() => { + expect(() => { + res.clone() + }).to.throw(Error) + }) + ) + }) + + xit('should timeout on cloning response without consuming one of the streams when the second packet size is equal default highWaterMark', function () { + this.timeout(300) + const url = local.mockState(res => { + // Observed behavior of TCP packets splitting: + // - response body size <= 65438 → single packet sent + // - response body size > 65438 → multiple packets sent + // Max TCP packet size is 64kB (http://stackoverflow.com/a/2614188/5763764), + // but first packet probably transfers more than the response body. + const firstPacketMaxSize = 65438 + const secondPacketSize = 16 * 1024 // = defaultHighWaterMark + res.end(crypto.randomBytes(firstPacketMaxSize + secondPacketSize)) + }) + return expect( + fetch(url).then(res => res.clone().buffer()) + ).to.timeout + }) + + xit('should timeout on cloning response without consuming one of the streams when the second packet size is equal custom highWaterMark', function () { + this.timeout(300) + const url = local.mockState(res => { + const firstPacketMaxSize = 65438 + const secondPacketSize = 10 + res.end(crypto.randomBytes(firstPacketMaxSize + secondPacketSize)) + }) + return expect( + fetch(url, { highWaterMark: 10 }).then(res => res.clone().buffer()) + ).to.timeout + }) + + xit('should not timeout on cloning response without consuming one of the streams when the second packet size is less than default highWaterMark', function () { + // TODO: fix test. + if (!isNodeLowerThan('v16.0.0')) { + this.skip() + } + + this.timeout(300) + const url = local.mockState(res => { + const firstPacketMaxSize = 65438 + const secondPacketSize = 16 * 1024 // = defaultHighWaterMark + res.end(crypto.randomBytes(firstPacketMaxSize + secondPacketSize - 1)) + }) + return expect( + fetch(url).then(res => res.clone().buffer()) + ).not.to.timeout + }) + + xit('should not timeout on cloning response without consuming one of the streams when the second packet size is less than custom highWaterMark', function () { + // TODO: fix test. + if (!isNodeLowerThan('v16.0.0')) { + this.skip() + } + + this.timeout(300) + const url = local.mockState(res => { + const firstPacketMaxSize = 65438 + const secondPacketSize = 10 + res.end(crypto.randomBytes(firstPacketMaxSize + secondPacketSize - 1)) + }) + return expect( + fetch(url, { highWaterMark: 10 }).then(res => res.clone().buffer()) + ).not.to.timeout + }) + + xit('should not timeout on cloning response without consuming one of the streams when the response size is double the custom large highWaterMark - 1', function () { + // TODO: fix test. + if (!isNodeLowerThan('v16.0.0')) { + this.skip() + } + + this.timeout(300) + const url = local.mockState(res => { + res.end(crypto.randomBytes((2 * 512 * 1024) - 1)) + }) + return expect( + fetch(url, { highWaterMark: 512 * 1024 }).then(res => res.clone().buffer()) + ).not.to.timeout + }) + + it('should allow get all responses of a header', () => { + const url = `${base}cookie` + return fetch(url).then(res => { + const expected = 'a=1, b=1' + expect(res.headers.get('set-cookie')).to.equal(expected) + expect(res.headers.get('Set-Cookie')).to.equal(expected) + }) + }) + + it('should support fetch with Request instance', () => { + const url = `${base}hello` + const request = new Request(url) + return fetch(request).then(res => { + expect(res.url).to.equal(url) + expect(res.ok).to.be.true + expect(res.status).to.equal(200) + }) + }) + + it('should support fetch with Node.js URL object', () => { + const url = `${base}hello` + const urlObject = new URL(url) + const request = new Request(urlObject) + return fetch(request).then(res => { + expect(res.url).to.equal(url) + expect(res.ok).to.be.true + expect(res.status).to.equal(200) + }) + }) + + it('should support fetch with WHATWG URL object', () => { + const url = `${base}hello` + const urlObject = new URL(url) + const request = new Request(urlObject) + return fetch(request).then(res => { + expect(res.url).to.equal(url) + expect(res.ok).to.be.true + expect(res.status).to.equal(200) + }) + }) + + it('should keep `?` sign in URL when no params are given', () => { + const url = `${base}question?` + const urlObject = new URL(url) + const request = new Request(urlObject) + return fetch(request).then(res => { + expect(res.url).to.equal(url) + expect(res.ok).to.be.true + expect(res.status).to.equal(200) + }) + }) + + it('if params are given, do not modify anything', () => { + const url = `${base}question?a=1` + const urlObject = new URL(url) + const request = new Request(urlObject) + return fetch(request).then(res => { + expect(res.url).to.equal(url) + expect(res.ok).to.be.true + expect(res.status).to.equal(200) + }) + }) + + it('should preserve the hash (#) symbol', () => { + const url = `${base}question?#` + const urlObject = new URL(url) + const request = new Request(urlObject) + return fetch(request).then(res => { + expect(res.url).to.equal(url) + expect(res.ok).to.be.true + expect(res.status).to.equal(200) + }) + }) + + it('should support reading blob as text', () => { + return new Response('hello') + .blob() + .then(blob => blob.text()) + .then(body => { + expect(body).to.equal('hello') + }) + }) + + it('should support reading blob as arrayBuffer', () => { + return new Response('hello') + .blob() + .then(blob => blob.arrayBuffer()) + .then(ab => { + const string = String.fromCharCode.apply(null, new Uint8Array(ab)) + expect(string).to.equal('hello') + }) + }) + + it('should support blob round-trip', () => { + const url = `${base}hello` + + let length + let type + + return fetch(url).then(res => res.blob()).then(async blob => { + const url = `${base}inspect` + length = blob.size + type = blob.type + return fetch(url, { + method: 'POST', + body: blob + }) + }).then(res => res.json()).then(({ body, headers }) => { + expect(body).to.equal('world') + expect(headers['content-type']).to.equal(type) + expect(headers['content-length']).to.equal(String(length)) + }) + }) + + it('should support overwrite Request instance', () => { + const url = `${base}inspect` + const request = new Request(url, { + method: 'POST', + headers: { + a: '1' + } + }) + return fetch(request, { + method: 'GET', + headers: { + a: '2' + } + }).then(res => { + return res.json() + }).then(body => { + expect(body.method).to.equal('GET') + expect(body.headers.a).to.equal('2') + }) + }) + + it('should support http request', function () { + this.timeout(5000) + const url = 'https://github.com/' + const options = { + method: 'HEAD' + } + return fetch(url, options).then(res => { + expect(res.status).to.equal(200) + expect(res.ok).to.be.true + }) + }) + + it('should encode URLs as UTF-8', async () => { + const url = `${base}möbius` + const res = await fetch(url) + expect(res.url).to.equal(`${base}m%C3%B6bius`) + }) +}) diff --git a/test/node-fetch/request.js b/test/node-fetch/request.js new file mode 100644 index 00000000000..ea82d6add1c --- /dev/null +++ b/test/node-fetch/request.js @@ -0,0 +1,259 @@ +const stream = require('stream') +const http = require('http') + +const AbortController = require('abort-controller') +const chai = require('chai') +const { Blob } = require('buffer') + +const Request = require('../../lib/api/api-fetch/request.js').Request +const TestServer = require('./utils/server.js') + +const { expect } = chai + +describe('Request', () => { + const local = new TestServer() + let base + + before(async () => { + await local.start() + base = `http://${local.hostname}:${local.port}/` + }) + + after(async () => { + return local.stop() + }) + + it('should have attributes conforming to Web IDL', () => { + const request = new Request('http://github.com/') + const enumerableProperties = [] + for (const property in request) { + enumerableProperties.push(property) + } + + for (const toCheck of [ + 'body', + 'bodyUsed', + 'arrayBuffer', + 'blob', + 'json', + 'text', + 'method', + 'url', + 'headers', + 'redirect', + 'clone', + 'signal' + ]) { + expect(enumerableProperties).to.contain(toCheck) + } + + // for (const toCheck of [ + // 'body', 'bodyUsed', 'method', 'url', 'headers', 'redirect', 'signal' + // ]) { + // expect(() => { + // request[toCheck] = 'abc' + // }).to.throw() + // } + }) + + // it('should support wrapping Request instance', () => { + // const url = `${base}hello` + + // const form = new FormData() + // form.append('a', '1') + // const { signal } = new AbortController() + + // const r1 = new Request(url, { + // method: 'POST', + // follow: 1, + // body: form, + // signal + // }) + // const r2 = new Request(r1, { + // follow: 2 + // }) + + // expect(r2.url).to.equal(url) + // expect(r2.method).to.equal('POST') + // expect(r2.signal).to.equal(signal) + // // Note that we didn't clone the body + // expect(r2.body).to.equal(form) + // expect(r1.follow).to.equal(1) + // expect(r2.follow).to.equal(2) + // expect(r1.counter).to.equal(0) + // expect(r2.counter).to.equal(0) + // }) + + xit('should override signal on derived Request instances', () => { + const parentAbortController = new AbortController() + const derivedAbortController = new AbortController() + const parentRequest = new Request(`${base}hello`, { + signal: parentAbortController.signal + }) + const derivedRequest = new Request(parentRequest, { + signal: derivedAbortController.signal + }) + expect(parentRequest.signal).to.equal(parentAbortController.signal) + expect(derivedRequest.signal).to.equal(derivedAbortController.signal) + }) + + xit('should allow removing signal on derived Request instances', () => { + const parentAbortController = new AbortController() + const parentRequest = new Request(`${base}hello`, { + signal: parentAbortController.signal + }) + const derivedRequest = new Request(parentRequest, { + signal: null + }) + expect(parentRequest.signal).to.equal(parentAbortController.signal) + expect(derivedRequest.signal).to.equal(null) + }) + + it('should throw error with GET/HEAD requests with body', () => { + expect(() => new Request(base, { body: '' })) + .to.throw(TypeError) + expect(() => new Request(base, { body: 'a' })) + .to.throw(TypeError) + expect(() => new Request(base, { body: '', method: 'HEAD' })) + .to.throw(TypeError) + expect(() => new Request(base, { body: 'a', method: 'HEAD' })) + .to.throw(TypeError) + expect(() => new Request(base, { body: 'a', method: 'get' })) + .to.throw(TypeError) + expect(() => new Request(base, { body: 'a', method: 'head' })) + .to.throw(TypeError) + }) + + it('should default to null as body', () => { + const request = new Request(base) + expect(request.body).to.equal(null) + return request.text().then(result => expect(result).to.equal('')) + }) + + it('should support parsing headers', () => { + const url = base + const request = new Request(url, { + headers: { + a: '1' + } + }) + expect(request.url).to.equal(url) + expect(request.headers.get('a')).to.equal('1') + }) + + it('should support arrayBuffer() method', () => { + const url = base + const request = new Request(url, { + method: 'POST', + body: 'a=1' + }) + expect(request.url).to.equal(url) + return request.arrayBuffer().then(result => { + expect(result).to.be.an.instanceOf(ArrayBuffer) + const string = String.fromCharCode.apply(null, new Uint8Array(result)) + expect(string).to.equal('a=1') + }) + }) + + it('should support text() method', () => { + const url = base + const request = new Request(url, { + method: 'POST', + body: 'a=1' + }) + expect(request.url).to.equal(url) + return request.text().then(result => { + expect(result).to.equal('a=1') + }) + }) + + it('should support json() method', () => { + const url = base + const request = new Request(url, { + method: 'POST', + body: '{"a":1}' + }) + expect(request.url).to.equal(url) + return request.json().then(result => { + expect(result.a).to.equal(1) + }) + }) + + it('should support blob() method', () => { + const url = base + const request = new Request(url, { + method: 'POST', + body: Buffer.from('a=1') + }) + expect(request.url).to.equal(url) + return request.blob().then(result => { + expect(result).to.be.an.instanceOf(Blob) + expect(result.size).to.equal(3) + expect(result.type).to.equal('') + }) + }) + + xit('should support clone() method', () => { + const url = base + const body = stream.Readable.from('a=1') + const agent = new http.Agent() + const { signal } = new AbortController() + const request = new Request(url, { + body, + method: 'POST', + redirect: 'manual', + headers: { + b: '2' + }, + follow: 3, + compress: false, + agent, + signal + }) + const cl = request.clone() + expect(cl.url).to.equal(url) + expect(cl.method).to.equal('POST') + expect(cl.redirect).to.equal('manual') + expect(cl.headers.get('b')).to.equal('2') + expect(cl.method).to.equal('POST') + // Clone body shouldn't be the same body + expect(cl.body).to.not.equal(body) + return Promise.all([cl.text(), request.text()]).then(results => { + expect(results[0]).to.equal('a=1') + expect(results[1]).to.equal('a=1') + }) + }) + + it('should support ArrayBuffer as body', () => { + const encoder = new TextEncoder() + const request = new Request(base, { + method: 'POST', + body: encoder.encode('a=1').buffer + }) + return request.text().then(result => { + expect(result).to.equal('a=1') + }) + }) + + it('should support Uint8Array as body', () => { + const encoder = new TextEncoder() + const request = new Request(base, { + method: 'POST', + body: encoder.encode('a=1') + }) + return request.text().then(result => { + expect(result).to.equal('a=1') + }) + }) + + it('should support DataView as body', () => { + const encoder = new TextEncoder() + const request = new Request(base, { + method: 'POST', + body: new DataView(encoder.encode('a=1').buffer) + }) + return request.text().then(result => { + expect(result).to.equal('a=1') + }) + }) +}) diff --git a/test/node-fetch/response.js b/test/node-fetch/response.js new file mode 100644 index 00000000000..62417f4b7ad --- /dev/null +++ b/test/node-fetch/response.js @@ -0,0 +1,218 @@ +/* eslint no-unused-expressions: "off" */ + +const chai = require('chai') +const stream = require('stream') +const { Response } = require('../../lib/api/api-fetch/response.js') +const TestServer = require('./utils/server.js') +const toWeb = require('./utils/toWeb.js') +const { Blob } = require('buffer') +const { kUrlList } = require('../../lib/api/api-fetch/symbols.js') + +const { expect } = chai + +describe('Response', () => { + const local = new TestServer() + let base + + before(async () => { + await local.start() + base = `http://${local.hostname}:${local.port}/` + }) + + after(async () => { + return local.stop() + }) + + it('should have attributes conforming to Web IDL', () => { + const res = new Response() + const enumerableProperties = [] + for (const property in res) { + enumerableProperties.push(property) + } + + for (const toCheck of [ + 'body', + 'bodyUsed', + 'arrayBuffer', + 'blob', + 'json', + 'text', + 'type', + 'url', + 'status', + 'ok', + 'redirected', + 'statusText', + 'headers', + 'clone' + ]) { + expect(enumerableProperties).to.contain(toCheck) + } + + // TODO + // for (const toCheck of [ + // 'body', + // 'bodyUsed', + // 'type', + // 'url', + // 'status', + // 'ok', + // 'redirected', + // 'statusText', + // 'headers' + // ]) { + // expect(() => { + // res[toCheck] = 'abc' + // }).to.throw() + // } + }) + + it('should support empty options', () => { + const res = new Response(toWeb(stream.Readable.from('a=1'))) + return res.text().then(result => { + expect(result).to.equal('a=1') + }) + }) + + it('should support parsing headers', () => { + const res = new Response(null, { + headers: { + a: '1' + } + }) + expect(res.headers.get('a')).to.equal('1') + }) + + it('should support text() method', () => { + const res = new Response('a=1') + return res.text().then(result => { + expect(result).to.equal('a=1') + }) + }) + + it('should support json() method', () => { + const res = new Response('{"a":1}') + return res.json().then(result => { + expect(result.a).to.equal(1) + }) + }) + + if (Blob) { + it('should support blob() method', () => { + const res = new Response('a=1', { + method: 'POST', + headers: { + 'Content-Type': 'text/plain' + } + }) + return res.blob().then(result => { + expect(result).to.be.an.instanceOf(Blob) + expect(result.size).to.equal(3) + expect(result.type).to.equal('text/plain') + }) + }) + } + + xit('should support clone() method', () => { + const body = stream.Readable.from('a=1') + const res = new Response(body, { + headers: { + a: '1' + }, + [kUrlList]: base, + status: 346, + statusText: 'production' + }) + const cl = res.clone() + expect(cl.headers.get('a')).to.equal('1') + expect(cl.type).to.equal('default') + expect(cl.url).to.equal(base) + expect(cl.status).to.equal(346) + expect(cl.statusText).to.equal('production') + expect(cl.ok).to.be.false + // Clone body shouldn't be the same body + expect(cl.body).to.not.equal(body) + return Promise.all([cl.text(), res.text()]).then(results => { + expect(results[0]).to.equal('a=1') + expect(results[1]).to.equal('a=1') + }) + }) + + it('should support stream as body', () => { + const body = toWeb(stream.Readable.from('a=1')) + const res = new Response(body) + return res.text().then(result => { + expect(result).to.equal('a=1') + }) + }) + + it('should support string as body', () => { + const res = new Response('a=1') + return res.text().then(result => { + expect(result).to.equal('a=1') + }) + }) + + it('should support buffer as body', () => { + const res = new Response(Buffer.from('a=1')) + return res.text().then(result => { + expect(result).to.equal('a=1') + }) + }) + + it('should support ArrayBuffer as body', () => { + const encoder = new TextEncoder() + const res = new Response(encoder.encode('a=1')) + return res.text().then(result => { + expect(result).to.equal('a=1') + }) + }) + + it('should support blob as body', async () => { + const res = new Response(new Blob(['a=1'])) + return res.text().then(result => { + expect(result).to.equal('a=1') + }) + }) + + it('should support Uint8Array as body', () => { + const encoder = new TextEncoder() + const res = new Response(encoder.encode('a=1')) + return res.text().then(result => { + expect(result).to.equal('a=1') + }) + }) + + it('should support DataView as body', () => { + const encoder = new TextEncoder() + const res = new Response(new DataView(encoder.encode('a=1').buffer)) + return res.text().then(result => { + expect(result).to.equal('a=1') + }) + }) + + it('should default to null as body', () => { + const res = new Response() + expect(res.body).to.equal(null) + + return res.text().then(result => expect(result).to.equal('')) + }) + + it('should default to 200 as status code', () => { + const res = new Response(null) + expect(res.status).to.equal(200) + }) + + it('should default to empty string as url', () => { + const res = new Response() + expect(res.url).to.equal('') + }) + + it('should support error() static method', () => { + const res = Response.error() + expect(res).to.be.an.instanceof(Response) + expect(res.type).to.equal('error') + expect(res.status).to.equal(0) + expect(res.statusText).to.equal('') + }) +}) diff --git a/test/node-fetch/utils/chai-timeout.js b/test/node-fetch/utils/chai-timeout.js new file mode 100644 index 00000000000..6838a4cc322 --- /dev/null +++ b/test/node-fetch/utils/chai-timeout.js @@ -0,0 +1,15 @@ +const pTimeout = require('p-timeout') + +module.exports = ({ Assertion }, utils) => { + utils.addProperty(Assertion.prototype, 'timeout', async function () { + let timeouted = false + await pTimeout(this._obj, 150, () => { + timeouted = true + }) + return this.assert( + timeouted, + 'expected promise to timeout but it was resolved', + 'expected promise not to timeout but it timed out' + ) + }) +} diff --git a/test/node-fetch/utils/dummy.txt b/test/node-fetch/utils/dummy.txt new file mode 100644 index 00000000000..5ca51916b39 --- /dev/null +++ b/test/node-fetch/utils/dummy.txt @@ -0,0 +1 @@ +i am a dummy \ No newline at end of file diff --git a/test/node-fetch/utils/read-stream.js b/test/node-fetch/utils/read-stream.js new file mode 100644 index 00000000000..7d791532934 --- /dev/null +++ b/test/node-fetch/utils/read-stream.js @@ -0,0 +1,9 @@ +module.exports = async function readStream (stream) { + const chunks = [] + + for await (const chunk of stream) { + chunks.push(chunk instanceof Buffer ? chunk : Buffer.from(chunk)) + } + + return Buffer.concat(chunks) +} diff --git a/test/node-fetch/utils/server.js b/test/node-fetch/utils/server.js new file mode 100644 index 00000000000..92426be3e15 --- /dev/null +++ b/test/node-fetch/utils/server.js @@ -0,0 +1,437 @@ +const http = require('http') +const zlib = require('zlib') +const { once } = require('events') +const Busboy = require('busboy') + +module.exports = class TestServer { + constructor () { + this.server = http.createServer(this.router) + // Node 8 default keepalive timeout is 5000ms + // make it shorter here as we want to close server quickly at the end of tests + this.server.keepAliveTimeout = 1000 + this.server.on('error', err => { + console.log(err.stack) + }) + this.server.on('connection', socket => { + socket.setTimeout(1500) + }) + } + + async start () { + this.server.listen(0, 'localhost') + return once(this.server, 'listening') + } + + async stop () { + this.server.close() + return once(this.server, 'close') + } + + get port () { + return this.server.address().port + } + + get hostname () { + return 'localhost' + } + + mockState (responseHandler) { + this.server.nextResponseHandler = responseHandler + return `http://${this.hostname}:${this.port}/mocked` + } + + router (request, res) { + const p = request.url + + if (p === '/mocked') { + if (this.nextResponseHandler) { + this.nextResponseHandler(res) + this.nextResponseHandler = undefined + } else { + throw new Error('No mocked response. Use ’TestServer.mockState()’.') + } + } + + if (p === '/hello') { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/plain') + res.end('world') + } + + if (p.includes('question')) { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/plain') + res.end('ok') + } + + if (p === '/plain') { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/plain') + res.end('text') + } + + if (p === '/no-status-text') { + res.writeHead(200, '', {}).end() + } + + if (p === '/options') { + res.statusCode = 200 + res.setHeader('Allow', 'GET, HEAD, OPTIONS') + res.end('hello world') + } + + if (p === '/html') { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/html') + res.end('') + } + + if (p === '/json') { + res.statusCode = 200 + res.setHeader('Content-Type', 'application/json') + res.end(JSON.stringify({ + name: 'value' + })) + } + + if (p === '/gzip') { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/plain') + res.setHeader('Content-Encoding', 'gzip') + zlib.gzip('hello world', (err, buffer) => { + if (err) { + throw err + } + + res.end(buffer) + }) + } + + if (p === '/gzip-truncated') { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/plain') + res.setHeader('Content-Encoding', 'gzip') + zlib.gzip('hello world', (err, buffer) => { + if (err) { + throw err + } + + // Truncate the CRC checksum and size check at the end of the stream + res.end(buffer.slice(0, -8)) + }) + } + + if (p === '/gzip-capital') { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/plain') + res.setHeader('Content-Encoding', 'GZip') + zlib.gzip('hello world', (err, buffer) => { + if (err) { + throw err + } + + res.end(buffer) + }) + } + + if (p === '/deflate') { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/plain') + res.setHeader('Content-Encoding', 'deflate') + zlib.deflate('hello world', (err, buffer) => { + if (err) { + throw err + } + + res.end(buffer) + }) + } + + if (p === '/brotli') { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/plain') + if (typeof zlib.createBrotliDecompress === 'function') { + res.setHeader('Content-Encoding', 'br') + zlib.brotliCompress('hello world', (err, buffer) => { + if (err) { + throw err + } + + res.end(buffer) + }) + } + } + + if (p === '/deflate-raw') { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/plain') + res.setHeader('Content-Encoding', 'deflate') + zlib.deflateRaw('hello world', (err, buffer) => { + if (err) { + throw err + } + + res.end(buffer) + }) + } + + if (p === '/sdch') { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/plain') + res.setHeader('Content-Encoding', 'sdch') + res.end('fake sdch string') + } + + if (p === '/invalid-content-encoding') { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/plain') + res.setHeader('Content-Encoding', 'gzip') + res.end('fake gzip string') + } + + if (p === '/timeout') { + setTimeout(() => { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/plain') + res.end('text') + }, 1000) + } + + if (p === '/slow') { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/plain') + res.write('test') + setTimeout(() => { + res.end('test') + }, 1000) + } + + if (p === '/cookie') { + res.statusCode = 200 + res.setHeader('Set-Cookie', ['a=1', 'b=1']) + res.end('cookie') + } + + if (p === '/size/chunk') { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/plain') + setTimeout(() => { + res.write('test') + }, 10) + setTimeout(() => { + res.end('test') + }, 20) + } + + if (p === '/size/long') { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/plain') + res.end('testtest') + } + + if (p === '/redirect/301') { + res.statusCode = 301 + res.setHeader('Location', '/inspect') + res.end() + } + + if (p === '/redirect/302') { + res.statusCode = 302 + res.setHeader('Location', '/inspect') + res.end() + } + + if (p === '/redirect/303') { + res.statusCode = 303 + res.setHeader('Location', '/inspect') + res.end() + } + + if (p === '/redirect/307') { + res.statusCode = 307 + res.setHeader('Location', '/inspect') + res.end() + } + + if (p === '/redirect/308') { + res.statusCode = 308 + res.setHeader('Location', '/inspect') + res.end() + } + + if (p === '/redirect/chain') { + res.statusCode = 301 + res.setHeader('Location', '/redirect/301') + res.end() + } + + if (p.startsWith('/redirect/chain/')) { + const count = parseInt(p.split('/').pop()) - 1 + res.statusCode = 301 + res.setHeader('Location', count ? `/redirect/chain/${count}` : '/redirect/301') + res.end() + } + + if (p === '/redirect/no-location') { + res.statusCode = 301 + res.end() + } + + if (p === '/redirect/slow') { + res.statusCode = 301 + res.setHeader('Location', '/redirect/301') + setTimeout(() => { + res.end() + }, 1000) + } + + if (p === '/redirect/slow-chain') { + res.statusCode = 301 + res.setHeader('Location', '/redirect/slow') + setTimeout(() => { + res.end() + }, 10) + } + + if (p === '/redirect/slow-stream') { + res.statusCode = 301 + res.setHeader('Location', '/slow') + res.end() + } + + if (p === '/redirect/bad-location') { + res.socket.write('HTTP/1.1 301\r\nLocation: ☃\r\nContent-Length: 0\r\n') + res.socket.end('\r\n') + } + + if (p === '/error/400') { + res.statusCode = 400 + res.setHeader('Content-Type', 'text/plain') + res.end('client error') + } + + if (p === '/error/404') { + res.statusCode = 404 + res.setHeader('Content-Encoding', 'gzip') + res.end() + } + + if (p === '/error/500') { + res.statusCode = 500 + res.setHeader('Content-Type', 'text/plain') + res.end('server error') + } + + if (p === '/error/reset') { + res.destroy() + } + + if (p === '/error/premature') { + res.writeHead(200, { 'content-length': 50 }) + res.write('foo') + setTimeout(() => { + res.destroy() + }, 100) + } + + if (p === '/error/premature/chunked') { + res.writeHead(200, { + 'Content-Type': 'application/json', + 'Transfer-Encoding': 'chunked' + }) + + res.write(`${JSON.stringify({ data: 'hi' })}\n`) + + setTimeout(() => { + res.write(`${JSON.stringify({ data: 'bye' })}\n`) + }, 200) + + setTimeout(() => { + res.destroy() + }, 400) + } + + if (p === '/error/json') { + res.statusCode = 200 + res.setHeader('Content-Type', 'application/json') + res.end('invalid json') + } + + if (p === '/no-content') { + res.statusCode = 204 + res.end() + } + + if (p === '/no-content/gzip') { + res.statusCode = 204 + res.setHeader('Content-Encoding', 'gzip') + res.end() + } + + if (p === '/no-content/brotli') { + res.statusCode = 204 + res.setHeader('Content-Encoding', 'br') + res.end() + } + + if (p === '/not-modified') { + res.statusCode = 304 + res.end() + } + + if (p === '/not-modified/gzip') { + res.statusCode = 304 + res.setHeader('Content-Encoding', 'gzip') + res.end() + } + + if (p === '/inspect') { + res.statusCode = 200 + res.setHeader('Content-Type', 'application/json') + let body = '' + request.on('data', c => { + body += c + }) + request.on('end', () => { + res.end(JSON.stringify({ + method: request.method, + url: request.url, + headers: request.headers, + body + })) + }) + } + + if (p === '/multipart') { + res.statusCode = 200 + res.setHeader('Content-Type', 'application/json') + const busboy = new Busboy({ headers: request.headers }) + let body = '' + busboy.on('file', async (fieldName, file, fileName) => { + body += `${fieldName}=${fileName}` + // consume file data + // eslint-disable-next-line no-empty, no-unused-vars + for await (const c of file) {} + }) + + busboy.on('field', (fieldName, value) => { + body += `${fieldName}=${value}` + }) + busboy.on('finish', () => { + res.end(JSON.stringify({ + method: request.method, + url: request.url, + headers: request.headers, + body + })) + }) + request.pipe(busboy) + } + + if (p === '/m%C3%B6bius') { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/plain') + res.end('ok') + } + } +} diff --git a/test/node-fetch/utils/toWeb.js b/test/node-fetch/utils/toWeb.js new file mode 100644 index 00000000000..3639146019d --- /dev/null +++ b/test/node-fetch/utils/toWeb.js @@ -0,0 +1,73 @@ +let ReadableStream +let CountQueuingStrategy + +const { destroy, isDestroyed } = require('../../../lib/core/util') + +const { finished } = require('stream') +const { AbortError } = require('../../../lib/core/errors') + +module.exports = function toWeb (streamReadable) { + if (!ReadableStream) { + ReadableStream = require('stream/web').ReadableStream + } + if (!CountQueuingStrategy) { + CountQueuingStrategy = require('stream/web').CountQueuingStrategy + } + + if (isDestroyed(streamReadable)) { + const readable = new ReadableStream() + readable.cancel() + return readable + } + + const objectMode = streamReadable.readableObjectMode + const highWaterMark = streamReadable.readableHighWaterMark + const strategy = objectMode + ? new CountQueuingStrategy({ highWaterMark }) + : { highWaterMark } + + let controller + + function onData (chunk) { + // Copy the Buffer to detach it from the pool. + if (Buffer.isBuffer(chunk) && !objectMode) { + chunk = new Uint8Array(chunk) + } + controller.enqueue(chunk) + if (controller.desiredSize <= 0) { + streamReadable.pause() + } + } + + streamReadable.pause() + + finished(streamReadable, (err) => { + if (err && err.code === 'ERR_STREAM_PREMATURE_CLOSE') { + const er = new AbortError() + er.cause = er + err = er + } + + if (err) { + controller.error(err) + } else { + controller.close() + } + }) + + streamReadable.on('data', onData) + + return new ReadableStream({ + start (c) { + controller = c + }, + + pull () { + streamReadable.resume() + }, + + cancel (reason) { + destroy(streamReadable, reason) + } + }, strategy) +} From 866cc904d0e0c22eb4b5e6480541adde9d3c85c0 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Wed, 11 Aug 2021 07:41:15 +0200 Subject: [PATCH 02/56] fixup: strict null check --- lib/api/api-fetch/body.js | 2 +- lib/api/api-fetch/index.js | 18 +++++++++--------- lib/api/api-fetch/request.js | 15 ++++++++------- lib/api/api-fetch/response.js | 2 +- test/node-fetch/main.js | 13 ++++--------- 5 files changed, 23 insertions(+), 27 deletions(-) diff --git a/lib/api/api-fetch/body.js b/lib/api/api-fetch/body.js index cddbb1b0d15..e98e2721254 100644 --- a/lib/api/api-fetch/body.js +++ b/lib/api/api-fetch/body.js @@ -55,7 +55,7 @@ function extractBody (body) { } if (!stream) { - assert(source != null) + assert(source !== null) if (!ReadableStream) { ReadableStream = require('stream/web').ReadableStream } diff --git a/lib/api/api-fetch/index.js b/lib/api/api-fetch/index.js index 9f9e82bd367..f9ee8860765 100644 --- a/lib/api/api-fetch/index.js +++ b/lib/api/api-fetch/index.js @@ -155,7 +155,7 @@ function abortFetch (p, request, responseObject) { // 3. If request’s body is not null and is readable, then cancel request’s // body with error. - if (request.body != null) { + if (request.body !== null) { // NOTE: We don't have any way to check if body is readable. However, // cancel will be noop if state is "closed" and reject if state is // "error", which means that calling cancel() and catch errors @@ -179,7 +179,7 @@ function abortFetch (p, request, responseObject) { // 6. If response’s body is not null and is readable, then error response’s // body with error. - if (response.body != null && context.controller) { + if (response.body !== null && context.controller) { context.controller.error(error) } } @@ -551,7 +551,7 @@ function fetchFinale (fetchParams, response) { // 1. If fetchParams’s process response is non-null, // then queue a fetch task to run fetchParams’s process response // given response, with fetchParams’s task destination. - if (fetchParams.processResponse != null) { + if (fetchParams.processResponse !== null) { fetchParams.processResponse(response) } @@ -713,7 +713,7 @@ function httpRedirectFetch (fetchParams, response) { // and request’s body’s source is null, then return a network error. if ( actualResponse.status !== 303 && - request.body != null && + request.body !== null && request.body.source == null ) { return makeNetworkError() @@ -750,7 +750,7 @@ function httpRedirectFetch (fetchParams, response) { // 14. If request’s body is non-null, then set request’s body to the first return // value of safely extracting request’s body’s source. - if (request.body != null) { + if (request.body !== null) { assert(request.body.source) request.body = safelyExtractBody(request.body.source)[0] } @@ -848,7 +848,7 @@ async function httpNetworkOrCacheFetch ( // 7. If contentLength is non-null, then set contentLengthHeaderValue to // contentLength, serialized and isomorphic encoded. - if (contentLength != null) { + if (contentLength !== null) { // TODO: isomorphic encoded contentLengthHeaderValue = String(contentLength) } @@ -856,13 +856,13 @@ async function httpNetworkOrCacheFetch ( // 8. If contentLengthHeaderValue is non-null, then append // `Content-Length`/contentLengthHeaderValue to httpRequest’s header // list. - if (contentLengthHeaderValue != null) { + if (contentLengthHeaderValue !== null) { httpRequest.headersList.append('content-length', contentLengthHeaderValue) } // 9. If contentLength is non-null and httpRequest’s keepalive is true, // then: - if (contentLength != null && httpRequest.keepalive) { + if (contentLength !== null && httpRequest.keepalive) { // NOTE: keepalive is a noop outside of browser context. } @@ -1041,7 +1041,7 @@ async function httpNetworkOrCacheFetch ( // isNewConnectionFetch is false !isNewConnectionFetch && // request’s body is null, or request’s body is non-null and request’s body’s source is non-null - (request.body == null || request.body.source != null) + (request.body == null || request.body.source !== null) ) { // 1. If the ongoing fetch is terminated, then: if (context.terminated) { diff --git a/lib/api/api-fetch/request.js b/lib/api/api-fetch/request.js index 9f2d93d36db..7a59b089e34 100644 --- a/lib/api/api-fetch/request.js +++ b/lib/api/api-fetch/request.js @@ -189,7 +189,7 @@ class Request { } // 18. If mode is non-null, set request’s mode to mode. - if (mode != null) { + if (mode !== null) { request.mode = mode } @@ -277,9 +277,10 @@ class Request { this[kSignal] = ac.signal // 29. If signal is not null, then make this’s signal follow signal. - if (signal != null) { + if (signal !== null) { if ( - typeof signal.aborted !== 'boolean' && + !signal || + typeof signal.aborted !== 'boolean' || typeof signal.addEventListener !== 'function') { throw new TypeError(`'signal' option '${signal}' is not a valid value of AbortSignal`) } @@ -343,7 +344,7 @@ class Request { // non-null, and request’s method is `GET` or `HEAD`, then throw a // TypeError. if ( - (('body' in init && init != null) || inputBody != null) && + (('body' in init && init !== null) || inputBody !== null) && (request.method === 'GET' || request.method === 'HEAD') ) { throw new TypeError() @@ -353,7 +354,7 @@ class Request { let initBody = null // 36. If init["body"] exists and is non-null, then: - if ('body' in init && init != null) { + if ('body' in init && init !== null) { // 1. Let Content-Type be null. // 2. Set initBody and Content-Type to the result of extracting // init["body"], with keepalive set to request’s keepalive. @@ -374,7 +375,7 @@ class Request { // 38. If inputOrInitBody is non-null and inputOrInitBody’s source is // null, then: - if (inputOrInitBody != null && inputOrInitBody.source == null) { + if (inputOrInitBody !== null && inputOrInitBody.source == null) { // 1. If this’s request’s mode is neither "same-origin" nor "cors", // then throw a TypeError. if (request.mode !== 'same-origin' && request.mode !== 'cors') { @@ -389,7 +390,7 @@ class Request { let finalBody = inputOrInitBody // 40. If initBody is null and inputBody is non-null, then: - if (initBody == null && inputBody != null) { + if (initBody == null && inputBody !== null) { // 1. If input is unusable, then throw a TypeError. if (util.isDisturbed(inputBody.stream) || inputBody.stream.locked) { throw new TypeError('unusable') diff --git a/lib/api/api-fetch/response.js b/lib/api/api-fetch/response.js index 855a6ee8ed9..fa7f5fb560e 100644 --- a/lib/api/api-fetch/response.js +++ b/lib/api/api-fetch/response.js @@ -121,7 +121,7 @@ class Response { } // 8. If body is non-null, then: - if (body != null) { + if (body !== null) { // 1. If init["status"] is a null body status, then throw a TypeError. if (nullBodyStatus.includes(init.status)) { throw new TypeError() diff --git a/test/node-fetch/main.js b/test/node-fetch/main.js index 62707936def..d64353ce11a 100644 --- a/test/node-fetch/main.js +++ b/test/node-fetch/main.js @@ -946,15 +946,10 @@ describe('node-fetch', () => { ]) }) - it('should gracefully handle a nullish signal', () => { - return Promise.all([ - fetch(`${base}hello`, { signal: null }).then(res => { - return expect(res.ok).to.be.true - }), - fetch(`${base}hello`, { signal: undefined }).then(res => { - return expect(res.ok).to.be.true - }) - ]) + it('should gracefully handle a null signal', () => { + return fetch(`${base}hello`, { signal: null }).then(res => { + return expect(res.ok).to.be.true + }) }) it('should allow setting User-Agent', () => { From 217af2bf6a4c7745dd1ef63eab542c254c0d1e39 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Wed, 11 Aug 2021 08:02:50 +0200 Subject: [PATCH 03/56] fixup: extractBody --- lib/api/api-fetch/body.js | 154 +++++++++++++++++++++++++---------- lib/api/api-fetch/request.js | 2 +- 2 files changed, 111 insertions(+), 45 deletions(-) diff --git a/lib/api/api-fetch/body.js b/lib/api/api-fetch/body.js index e98e2721254..ca4e640c75f 100644 --- a/lib/api/api-fetch/body.js +++ b/lib/api/api-fetch/body.js @@ -4,74 +4,140 @@ const util = require('../../core/util') const { kState } = require('./symbols') const { Blob } = require('buffer') const { NotSupportedError } = require('../../core/errors') -const assert = require('assert') - -let ReadableStream +const { ReadableStream } = require('stream/web') // https://fetch.spec.whatwg.org/#concept-bodyinit-extract -function extractBody (body) { - // TODO: FormBody +function extractBody (object, keepalive = false) { + // 1. Let stream be object if object is a ReadableStream object. + // Otherwise, let stream be a new ReadableStream, and set up stream. + let stream = object + let controller + if (!stream || typeof stream.pipeThrough !== 'function') { + stream = new ReadableStream({ + async start (c) { + controller = c + }, + async pull () { + }, + async cancel (reason) { + } + }) + } + + // 2. Let action be null. + let action = null - let stream = null + // 3. Let source be null. let source = null + + // 4. Let length be null. let length = null + + // 5. Let Content-Type be null. let type = null - if (body == null) { - return [null, null] - } else if (body instanceof URLSearchParams) { + // 6. Switch on object: + if (object instanceof URLSearchParams) { // spec says to run application/x-www-form-urlencoded on body.list // this is implemented in Node.js as apart of an URLSearchParams instance toString method // See: https://github.com/nodejs/node/blob/e46c680bf2b211bbd52cf959ca17ee98c7f657f5/lib/internal/url.js#L490 // and https://github.com/nodejs/node/blob/e46c680bf2b211bbd52cf959ca17ee98c7f657f5/lib/internal/url.js#L1100 - source = body.toString() - length = Buffer.byteLength(source) + + // Set source to the result of running the application/x-www-form-urlencoded serializer with object’s list. + source = object.toString() + + // Set Content-Type to `application/x-www-form-urlencoded;charset=UTF-8`. type = 'application/x-www-form-urlencoded;charset=UTF-8' - } else if (body instanceof ArrayBuffer || ArrayBuffer.isView(body)) { - if (body instanceof DataView) { + } else if (object instanceof ArrayBuffer || ArrayBuffer.isView(object)) { + if (object instanceof DataView) { // TODO: Blob doesn't seem to work with DataView? - body = body.buffer + object = object.buffer + } + + // Set source to a copy of the bytes held by object. + source = util.isBuffer(object) + ? Buffer.from(object) // Buffer.slice references same memory. + : object.slice(0) + } else if (util.isBlob(object)) { + // Set action to this step: read object. + action = async (onNext, onError, onComplete) => { + try { + onNext(await object.arrayBuffer()) + onComplete() + } catch (err) { + onError(err) + } + } + + // Set source to object. + source = object + + // Set length to object’s size. + length = object.size + + // If object’s type attribute is not the empty byte sequence, set Content-Type to its value. + if (object.type) { + type = object.type } - source = body - length = Buffer.byteLength(body) - } else if (util.isBlob(body)) { - source = body - length = body.size - type = body.type - } else if (typeof body.pipeThrough === 'function') { - if (util.isDisturbed(body)) { + } else if (typeof object.pipeThrough === 'function') { + // If keepalive is true, then throw a TypeError. + if (keepalive) { + throw new TypeError('keepalive') + } + + // If object is disturbed or locked, then throw a TypeError. + if (util.isDisturbed(stream)) { throw new TypeError('disturbed') } - if (body.locked) { + if (stream.locked) { throw new TypeError('locked') } - - stream = body + } else if (typeof object === 'string') { + source = object + type = 'text/plain;charset=UTF-8' } else { - source = String(body) - length = Buffer.byteLength(source) + // TODO: byte sequence? + // TODO: FormData? + // TODO: else? + source = String(object) type = 'text/plain;charset=UTF-8' } - if (!stream) { - assert(source !== null) - if (!ReadableStream) { - ReadableStream = require('stream/web').ReadableStream - } - stream = new ReadableStream({ - async start (controller) { - controller.enqueue(source) - controller.close() - }, - async pull () { - }, - async cancel (reason) { - } + // 7. If source is a byte sequence, then set action to a + // step that returns source and length to source’s length. + // TODO: What is a "byte sequence?" + if (typeof source === 'string') { + length = Buffer.byteLength(source) + } + + // 8. If action is non-null, then run these steps in in parallel: + if (action !== null) { + // Run action. + action(bytes => { + // Whenever one or more bytes are available and stream is not errored, + // enqueue a Uint8Array wrapping an ArrayBuffer containing the available + // bytes into stream. + controller.enqueue(new Uint8Array(bytes)) + }, err => { + // TODO: Spec doesn't say anything about this? + controller.error(err) + }, () => { + // When running action is done, close stream. + controller.close() }) + } else if (controller) { + // TODO: Spec doesn't say anything about this? + controller.enqueue(source) + controller.close() } - return [{ stream, source, length }, type] + // 9. Let body be a body whose stream is stream, source is source, + // and length is length. + const body = { stream, source, length } + + // 10. Return body and Content-Type. + return [body, type] } function cloneBody (src) { @@ -97,7 +163,7 @@ function cloneBody (src) { } } -function safelyExtractBody (body) { +function safelyExtractBody (body, keepalive = false) { if (util.isDisturbed(body)) { throw new TypeError('disturbed') } @@ -106,7 +172,7 @@ function safelyExtractBody (body) { throw new TypeError('locked') } - return extractBody(body) + return extractBody(body, keepalive) } const methods = { diff --git a/lib/api/api-fetch/request.js b/lib/api/api-fetch/request.js index 7a59b089e34..b3dcbf2c8df 100644 --- a/lib/api/api-fetch/request.js +++ b/lib/api/api-fetch/request.js @@ -358,7 +358,7 @@ class Request { // 1. Let Content-Type be null. // 2. Set initBody and Content-Type to the result of extracting // init["body"], with keepalive set to request’s keepalive. - const [extractedBody, contentType] = extractBody(init.body) + const [extractedBody, contentType] = extractBody(init.body, request.keepalive) initBody = extractedBody // 3, If Content-Type is non-null and this’s headers’s header list does From 0d40f36db1a50455535b8f3c4ff9141dfa05931c Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Wed, 11 Aug 2021 08:12:03 +0200 Subject: [PATCH 04/56] fixup: body used --- lib/api/api-fetch/body.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/api/api-fetch/body.js b/lib/api/api-fetch/body.js index ca4e640c75f..db40eeec09c 100644 --- a/lib/api/api-fetch/body.js +++ b/lib/api/api-fetch/body.js @@ -5,6 +5,7 @@ const { kState } = require('./symbols') const { Blob } = require('buffer') const { NotSupportedError } = require('../../core/errors') const { ReadableStream } = require('stream/web') +const { kBodyUsed } = require('../../core/symbols') // https://fetch.spec.whatwg.org/#concept-bodyinit-extract function extractBody (object, keepalive = false) { @@ -184,7 +185,9 @@ const methods = { throw new TypeError('unusable') } - this[kState].bodyUsed = true + // NOTE: stream.isDisturbed hasn't landed on Node 16.x yet. + this[kState].body.stream[kBodyUsed] = true + for await (const chunk of this[kState].body.stream) { chunks.push(chunk) } @@ -223,10 +226,7 @@ const properties = { bodyUsed: { enumerable: true, get () { - return this[kState].body && ( - util.isDisturbed(this[kState].body.stream) || - !!this[kState].bodyUsed - ) + return this[kState].body && util.isDisturbed(this[kState].body.stream) } } } From ca62f2a9e2d8952cafffd9fcc75c8e8a79110e6d Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Wed, 11 Aug 2021 08:13:05 +0200 Subject: [PATCH 05/56] fixup: use instanceof Blob --- lib/api/api-fetch/body.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/api/api-fetch/body.js b/lib/api/api-fetch/body.js index db40eeec09c..d422bb7bac3 100644 --- a/lib/api/api-fetch/body.js +++ b/lib/api/api-fetch/body.js @@ -59,7 +59,7 @@ function extractBody (object, keepalive = false) { source = util.isBuffer(object) ? Buffer.from(object) // Buffer.slice references same memory. : object.slice(0) - } else if (util.isBlob(object)) { + } else if (object instanceof Blob) { // Set action to this step: read object. action = async (onNext, onError, onComplete) => { try { From 2a6cdb19ae61bf6a964630e54b148261dbf2c9bc Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Wed, 11 Aug 2021 08:19:03 +0200 Subject: [PATCH 06/56] fixup --- lib/api/api-fetch/body.js | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/lib/api/api-fetch/body.js b/lib/api/api-fetch/body.js index d422bb7bac3..6f6bab5c8e5 100644 --- a/lib/api/api-fetch/body.js +++ b/lib/api/api-fetch/body.js @@ -13,7 +13,7 @@ function extractBody (object, keepalive = false) { // Otherwise, let stream be a new ReadableStream, and set up stream. let stream = object let controller - if (!stream || typeof stream.pipeThrough !== 'function') { + if (!stream || !(stream instanceof ReadableStream)) { stream = new ReadableStream({ async start (c) { controller = c @@ -35,10 +35,12 @@ function extractBody (object, keepalive = false) { let length = null // 5. Let Content-Type be null. - let type = null + let contentType = null // 6. Switch on object: if (object instanceof URLSearchParams) { + // URLSearchParams + // spec says to run application/x-www-form-urlencoded on body.list // this is implemented in Node.js as apart of an URLSearchParams instance toString method // See: https://github.com/nodejs/node/blob/e46c680bf2b211bbd52cf959ca17ee98c7f657f5/lib/internal/url.js#L490 @@ -48,8 +50,10 @@ function extractBody (object, keepalive = false) { source = object.toString() // Set Content-Type to `application/x-www-form-urlencoded;charset=UTF-8`. - type = 'application/x-www-form-urlencoded;charset=UTF-8' + contentType = 'application/x-www-form-urlencoded;charset=UTF-8' } else if (object instanceof ArrayBuffer || ArrayBuffer.isView(object)) { + // BufferSource + if (object instanceof DataView) { // TODO: Blob doesn't seem to work with DataView? object = object.buffer @@ -60,6 +64,8 @@ function extractBody (object, keepalive = false) { ? Buffer.from(object) // Buffer.slice references same memory. : object.slice(0) } else if (object instanceof Blob) { + // Blob + // Set action to this step: read object. action = async (onNext, onError, onComplete) => { try { @@ -78,9 +84,11 @@ function extractBody (object, keepalive = false) { // If object’s type attribute is not the empty byte sequence, set Content-Type to its value. if (object.type) { - type = object.type + contentType = object.type } - } else if (typeof object.pipeThrough === 'function') { + } else if (object instanceof ReadableStream) { + // ReadableStream + // If keepalive is true, then throw a TypeError. if (keepalive) { throw new TypeError('keepalive') @@ -95,14 +103,16 @@ function extractBody (object, keepalive = false) { throw new TypeError('locked') } } else if (typeof object === 'string') { + // scalar value string + // TODO: How to check for "scalar value string"? source = object - type = 'text/plain;charset=UTF-8' + contentType = 'text/plain;charset=UTF-8' } else { // TODO: byte sequence? // TODO: FormData? // TODO: else? source = String(object) - type = 'text/plain;charset=UTF-8' + contentType = 'text/plain;charset=UTF-8' } // 7. If source is a byte sequence, then set action to a @@ -119,6 +129,7 @@ function extractBody (object, keepalive = false) { // Whenever one or more bytes are available and stream is not errored, // enqueue a Uint8Array wrapping an ArrayBuffer containing the available // bytes into stream. + // TODO: stream is not errored? controller.enqueue(new Uint8Array(bytes)) }, err => { // TODO: Spec doesn't say anything about this? @@ -138,7 +149,7 @@ function extractBody (object, keepalive = false) { const body = { stream, source, length } // 10. Return body and Content-Type. - return [body, type] + return [body, contentType] } function cloneBody (src) { From 311339236c139d7dbaff0d4d4330ce3e881084ad Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Wed, 11 Aug 2021 08:23:28 +0200 Subject: [PATCH 07/56] fixup --- lib/api/api-fetch/body.js | 65 ++++++++++++++++------------- lib/api/api-fetch/headers.js | 75 ++++++++++++++++++---------------- lib/api/api-fetch/index.js | 77 +++++++++++++++++++---------------- lib/api/api-fetch/request.js | 29 ++++++++++--- lib/api/api-fetch/response.js | 42 ++++++++++++++----- lib/api/api-fetch/util.js | 67 +++++++++++++++++++++++++----- lib/client.js | 16 ++++++-- lib/core/request.js | 4 +- lib/handler/redirect.js | 4 +- test/node-fetch/headers.js | 4 +- test/node-fetch/main.js | 38 ++--------------- 11 files changed, 253 insertions(+), 168 deletions(-) diff --git a/lib/api/api-fetch/body.js b/lib/api/api-fetch/body.js index 6f6bab5c8e5..566d0530370 100644 --- a/lib/api/api-fetch/body.js +++ b/lib/api/api-fetch/body.js @@ -6,6 +6,7 @@ const { Blob } = require('buffer') const { NotSupportedError } = require('../../core/errors') const { ReadableStream } = require('stream/web') const { kBodyUsed } = require('../../core/symbols') +const assert = require('assert') // https://fetch.spec.whatwg.org/#concept-bodyinit-extract function extractBody (object, keepalive = false) { @@ -152,39 +153,39 @@ function extractBody (object, keepalive = false) { return [body, contentType] } -function cloneBody (src) { - if (!src) { - return - } +// https://fetch.spec.whatwg.org/#bodyinit-safely-extract +function safelyExtractBody (object, keepalive = false) { + // To safely extract a body and a `Content-Type` value from + // a byte sequence or BodyInit object object, run these steps: - if (util.isDisturbed(src)) { - throw new TypeError('disturbed') + // 1. If object is a ReadableStream object, then: + if (object instanceof ReadableStream) { + // Assert: object is neither disturbed nor locked. + assert(!util.isDisturbed(object), 'disturbed') + assert(!object.locked, 'locked') } - if (src.locked) { - throw new TypeError('locked') - } + // 2. Return the results of extracting object. + return extractBody(object, keepalive) +} + +function cloneBody (body) { + // To clone a body body, run these steps: // https://fetch.spec.whatwg.org/#concept-body-clone - const [out1, out2] = src.stream.tee() - src.stream = out1 - return { - stream: out2, - length: src.length, - source: src.source - } -} -function safelyExtractBody (body, keepalive = false) { - if (util.isDisturbed(body)) { - throw new TypeError('disturbed') - } + // 1. Let « out1, out2 » be the result of teeing body’s stream. + const [out1, out2] = body.stream.tee() - if (body && body.locked) { - throw new TypeError('locked') - } + // 2. Set body’s stream to out1. + body.stream = out1 - return extractBody(body, keepalive) + // 3. Return a body whose stream is out2 and other members are copied from body. + return { + stream: out2, + length: body.length, + source: body.source + } } const methods = { @@ -192,14 +193,20 @@ const methods = { const chunks = [] if (this[kState].body) { - if (this.bodyUsed || this[kState].body.stream.locked) { - throw new TypeError('unusable') + const stream = this[kState].body.stream + + if (util.isDisturbed(stream)) { + throw new TypeError('disturbed') + } + + if (stream.locked) { + throw new TypeError('locked') } // NOTE: stream.isDisturbed hasn't landed on Node 16.x yet. - this[kState].body.stream[kBodyUsed] = true + stream[kBodyUsed] = true - for await (const chunk of this[kState].body.stream) { + for await (const chunk of stream) { chunks.push(chunk) } } diff --git a/lib/api/api-fetch/headers.js b/lib/api/api-fetch/headers.js index d020541c0f7..e9af0743c41 100644 --- a/lib/api/api-fetch/headers.js +++ b/lib/api/api-fetch/headers.js @@ -2,7 +2,6 @@ 'use strict' -const { types } = require('util') const { validateHeaderName, validateHeaderValue } = require('http') const { kHeadersList } = require('../../core/symbols') const { kGuard } = require('./symbols') @@ -57,40 +56,44 @@ function isHeaders (object) { } function fill (headers, object) { - if (isHeaders(object)) { - // Object is instance of Headers - headers[kHeadersList].push(...object[kHeadersList]) - } else if (Array.isArray(object)) { - // Support both 1D and 2D arrays of header entries - if (Array.isArray(object[0])) { - // Array of arrays - for (let i = 0; i < object.length; i++) { - if (object[i].length !== 2) { - throw new TypeError(`The argument 'init' is not of length 2. Received ${object[i]}`) - } - headers.append(object[i][0], object[i][1]) + // To fill a Headers object headers with a given object object, run these steps: + + if (object[Symbol.iterator]) { + // 1. If object is a sequence, then for each header in object: + // TODO: How to check if sequence? + for (let header of object) { + // 1. If header does not contain exactly two items, then throw a TypeError. + if (!header[Symbol.iterator]) { + // TODO: Spec doesn't define what to do here? + throw new TypeError() } - } else if (typeof object[0] === 'string' || Buffer.isBuffer(object[0])) { - // Flat array of strings or Buffers - if (object.length % 2 !== 0) { - throw new TypeError(`The argument 'init' is not even in length. Received ${object}`) + + if (typeof header === 'string') { + // TODO: Spec doesn't define what to do here? + throw new TypeError() } - for (let i = 0; i < object.length; i += 2) { - headers.append( - object[i].toString('utf-8'), - object[i + 1].toString('utf-8') - ) + + if (!Array.isArray(header)) { + header = [...header] } - } else { - // All other array based entries - throw new TypeError(`The argument 'init' is not a valid array entry. Received ${object}`) + + if (header.length !== 2) { + throw new TypeError() + } + + // 2. Append header’s first item/header’s second item to headers. + headers.append(header[0], header[1]) } - } else if (!types.isBoxedPrimitive(object)) { - // Object of key/value entries - const entries = Object.entries(object) - for (let i = 0; i < entries.length; i++) { - headers.append(entries[i][0], entries[i][1]) + } else if (object && typeof object === 'object') { + // Otherwise, object is a record, then for each key → value in object, + // append key/value to headers. + // TODO: How to check if record? + for (const header of Object.entries(object)) { + headers.append(header[0], header[1]) } + } else { + // TODO: Spec doesn't define what to do here? + throw TypeError() } } @@ -178,13 +181,15 @@ class HeadersList extends Array { class Headers { constructor (init = {}) { - // validateObject allowArray = true - if (!Array.isArray(init) && typeof init !== 'object') { - throw new TypeError('The argument \'init\' must be one of type Object or Array') - } this[kHeadersList] = new HeadersList() + + // The new Headers(init) constructor steps are: + + // 1. Set this’s guard to "none". this[kGuard] = 'none' - fill(this, init) + + // 2. If init is given, then fill this with init. + fill(this[kHeadersList], init) } get [Symbol.toStringTag] () { diff --git a/lib/api/api-fetch/index.js b/lib/api/api-fetch/index.js index f9ee8860765..b4769a9dcd7 100644 --- a/lib/api/api-fetch/index.js +++ b/lib/api/api-fetch/index.js @@ -5,12 +5,11 @@ const { Response, makeNetworkError, filterResponse } = require('./response') const { Headers } = require('./headers') const { Request, makeRequest } = require('./request') -const { badPort } = require('./util') +const { requestBadPort, responseLocationURL, requestCurrentURL } = require('./util') const { kState, kHeaders, kGuard } = require('./symbols') const { AbortError } = require('../../core/errors') const assert = require('assert') const { safelyExtractBody } = require('./body') -const { STATUS_CODES } = require('http') const { redirectStatus, nullBodyStatus, @@ -26,6 +25,7 @@ async function fetch (resource, init) { const context = { dispatcher: this, abort: null, + controller: null, terminated: false, terminate ({ aborted } = {}) { if (this.terminated) { @@ -175,11 +175,9 @@ function abortFetch (p, request, responseObject) { } // 5. Let response be responseObject’s response. - const response = responseObject[kState] - // 6. If response’s body is not null and is readable, then error response’s // body with error. - if (response.body !== null && context.controller) { + if (context.controller) { context.controller.error(error) } } @@ -235,7 +233,7 @@ function fetching ({ // client’s origin. if (request.origin === 'client') { // TODO: What is correct here? - request.origin = request.currentURL.origin + request.origin = requestCurrentURL(request).origin } // 10. If request’s policy container is "client", then: @@ -304,7 +302,7 @@ async function mainFetch (fetchParams, recursive = false) { // 6. If should request be blocked due to a bad port, should fetching request // be blocked as mixed content, or should request be blocked by Content // Security Policy returns blocked, then set response to a network error. - if (badPort(request) === 'blocked') { + if (requestBadPort(request) === 'blocked') { return makeNetworkError('bad port') } // TODO: should fetching request be blocked as mixed content? @@ -378,7 +376,7 @@ async function mainFetch (fetchParams, recursive = false) { } // request’s current URL’s scheme is not an HTTP(S) scheme - if (!/^https?:/.test(request.currentURL.protocol)) { + if (!/^https?:/.test(requestCurrentURL(request).protocol)) { // Return a network error. return makeNetworkError('URL scheme must be a HTTP(S) scheme') } @@ -623,7 +621,10 @@ async function httpFetch (fetchParams) { // 1. If actualResponse’s status is not 303, request’s body is not null, // and the connection uses HTTP/2, then user agents may, and are even // encouraged to, transmit an RST_STREAM frame. - // NOTE: HTTP/2 is not supported. + // See, https://github.com/whatwg/fetch/issues/1288 + if (context.abort) { + context.abort() + } // 2. Switch on request’s redirect mode: if (request.redirect === 'error') { @@ -650,7 +651,7 @@ async function httpFetch (fetchParams) { } // https://fetch.spec.whatwg.org/#http-redirect-fetch -function httpRedirectFetch (fetchParams, response) { +async function httpRedirectFetch (fetchParams, response) { // 1. Let request be fetchParams’s request. const request = fetchParams.request @@ -662,17 +663,17 @@ function httpRedirectFetch (fetchParams, response) { // 3. Let locationURL be actualResponse’s location URL given request’s current // URL’s fragment. - let locationURL = actualResponse.headersList.get('location') + let locationURL - // 4. If locationURL is null, then return response. - if (locationURL == null) { - return response - } - - // 5. If locationURL is failure, then return a network error. try { - locationURL = new URL(locationURL, request.currentURL) + locationURL = responseLocationURL(actualResponse, requestCurrentURL(request).hash) + + // 4. If locationURL is null, then return response. + if (locationURL == null) { + return response + } } catch (err) { + // 5. If locationURL is failure, then return a network error. return makeNetworkError(err) } @@ -723,7 +724,7 @@ function httpRedirectFetch (fetchParams, response) { // origin and request’s origin is not same origin with request’s current // URL’s origin, then set request’s tainted origin flag. if ( - locationURL.origin !== request.currentURL.origin && + locationURL.origin !== requestCurrentURL(request).origin && request.origin !== locationURL.origin ) { request.taintedOrigin = true @@ -768,7 +769,6 @@ function httpRedirectFetch (fetchParams, response) { // 18. Append locationURL to request’s URL list. request.urlList.push(locationURL) - request.currentURL = locationURL // 19. Invoke set request’s referrer policy on redirect on request and actualResponse. // TODO @@ -936,7 +936,7 @@ async function httpNetworkOrCacheFetch ( // 18. Modify httpRequest’s header list per HTTP. Do not append a given // header if httpRequest’s header list contains that header’s name. - // TODO + // TODO: https://github.com/whatwg/fetch/issues/1285#issuecomment-896560129 // 19. If includeCredentials is true, then: if (includeCredentials) { @@ -1121,9 +1121,19 @@ function httpNetworkFetch ( } const context = this + + // TODO: forceNewConnection + + // NOTE: This is just a hack. + if (forceNewConnection && context.controller) { + context.abort() + } + + assert(!context.controller) + return new Promise((resolve) => context.dispatcher.dispatch({ - path: request.currentURL.pathname + request.currentURL.search, - origin: request.currentURL.origin, + path: requestCurrentURL(request).pathname + requestCurrentURL(request).search, + origin: requestCurrentURL(request).origin, method: request.method, body, headers: request.headersList, @@ -1137,18 +1147,21 @@ function httpNetworkFetch ( } }, - onHeaders (statusCode, headers, resume) { - if (statusCode < 200) { + onHeaders (status, headersList, resume, statusText) { + if (status < 200) { return } - headers = new Headers(headers) + const headers = new Headers() + for (let n = 0; n < headersList.length; n += 2) { + headers.append(headersList[n + 0].toString(), headersList[n + 1].toString()) + } if (!ReadableStream) { ReadableStream = require('stream/web').ReadableStream } - const stream = statusCode === 204 + const stream = status === 204 ? null : new ReadableStream({ async start (controller) { @@ -1171,13 +1184,9 @@ function httpNetworkFetch ( } }, { highWaterMark: 16384 }) - // TODO: Not sure we should use new Response here. - response = new Response(stream, { - status: statusCode, - // TODO: Get statusText from response? - statusText: STATUS_CODES[statusCode], - headers - })[kState] + // TODO: Not sure we should use new Response here? + response = new Response(stream, { status, statusText })[kState] + response.headersList.push(...headers[kHeadersList]) resolve(response) diff --git a/lib/api/api-fetch/request.js b/lib/api/api-fetch/request.js index b3dcbf2c8df..72d84928840 100644 --- a/lib/api/api-fetch/request.js +++ b/lib/api/api-fetch/request.js @@ -332,8 +332,12 @@ class Request { // 4. If headers is a Headers object, then for each header in its header // list, append header’s name/header’s value to this’s headers. - // 5. Otherwise, fill this’s headers with headers. - fillHeaders(this[kHeaders], headers) + if (headers instanceof Headers) { + this[kState].headersList.push(...headers[kHeadersList]) + } else { + // 5. Otherwise, fill this’s headers with headers. + fillHeaders(this[kState].headersList, headers) + } } // 33. Let inputBody be input’s request’s body if input is a Request @@ -561,8 +565,7 @@ class Request { } // 2. Let clonedRequest be the result of cloning this’s request. - const clonedRequest = makeRequest(this[kState]) - clonedRequest.body = cloneBody(clonedRequest.body) + const clonedRequest = cloneRequest(this[kState]) // 3. Let clonedRequestObject be the result of creating a Request object, // given clonedRequest, this’s headers’s guard, and this’s relevant Realm. @@ -636,11 +639,27 @@ function makeRequest (init) { ? [...init.urlList.map(url => new URL(url))] : [] } - request.currentURL = request.urlList[request.urlList.length - 1] request.url = request.urlList[0] return request } +// https://fetch.spec.whatwg.org/#concept-request-clone +function cloneRequest (request) { + // To clone a request request, run these steps: + + // 1. Let newRequest be a copy of request, except for its body. + const newRequest = makeRequest({ ...request, body: null }) + + // 2. If request’s body is non-null, set newRequest’s body to the + // result of cloning request’s body. + if (request.body !== null) { + newRequest.body = cloneBody(request.body) + } + + // 3. Return newRequest. + return newRequest +} + Object.defineProperties(Request.prototype, { method: kEnumerableProperty, url: kEnumerableProperty, diff --git a/lib/api/api-fetch/response.js b/lib/api/api-fetch/response.js index fa7f5fb560e..7eef9e0f98e 100644 --- a/lib/api/api-fetch/response.js +++ b/lib/api/api-fetch/response.js @@ -4,6 +4,7 @@ const { Headers, HeadersList, fill } = require('./headers') const { extractBody, cloneBody, mixinBody } = require('./body') const util = require('../../core/util') const { kEnumerableProperty } = util +const { responseURL } = require('./util') const { redirectStatus, nullBodyStatus, forbiddenHeaderNames } = require('./constants') const assert = require('assert') const { @@ -117,7 +118,7 @@ class Response { // 7. If init["headers"] exists, then fill this’s headers with init["headers"]. if ('headers' in init) { - fill(this[kHeaders], init.headers) + fill(this[kState].headersList, init.headers) } // 8. If body is non-null, then: @@ -161,14 +162,7 @@ class Response { // The url getter steps are to return the empty string if this’s // response’s URL is null; otherwise this’s response’s URL, // serialized with exclude fragment set to true. - let url - - // https://fetch.spec.whatwg.org/#responses - // A response has an associated URL. It is a pointer to the last URL - // in response’s URL list and null if response’s URL list is empty. - const urlList = this[kState].urlList - const length = urlList.length - url = length === 0 ? null : urlList[length - 1].toString() + let url = responseURL(this[kState]) if (url == null) { return '' @@ -223,8 +217,7 @@ class Response { } // 2. Let clonedResponse be the result of cloning this’s response. - const clonedResponse = makeResponse(this[kState]) - clonedResponse.body = cloneBody(clonedResponse.body) + const clonedResponse = cloneResponse(this[kState]) // 3. Return the result of creating a Response object, given // clonedResponse, this’s headers’s guard, and this’s relevant Realm. @@ -250,6 +243,33 @@ Object.defineProperties(Response.prototype, { clone: kEnumerableProperty }) +// https://fetch.spec.whatwg.org/#concept-response-clone +function cloneResponse (response) { + // To clone a response response, run these steps: + + // 1. If response is a filtered response, then return a new identical + // filtered response whose internal response is a clone of response’s + // internal response. + if (response.internalResponse) { + return { + ...response, + internalResponse: cloneResponse(response.internalResponse) + } + } + + // 2. Let newResponse be a copy of response, except for its body. + const newResponse = makeResponse({ ...response, body: null }) + + // 3. If response’s body is non-null, then set newResponse’s body to the + // result of cloning response’s body. + if (response.body !== null) { + newResponse.body = cloneBody(response.body) + } + + // 4. Return newResponse. + return newResponse +} + function makeResponse (init) { return { internalResponse: null, diff --git a/lib/api/api-fetch/util.js b/lib/api/api-fetch/util.js index 749a072e45e..1d8c5817f16 100644 --- a/lib/api/api-fetch/util.js +++ b/lib/api/api-fetch/util.js @@ -1,5 +1,7 @@ 'use strict' +const { redirectStatus } = require('./constants') + const badPorts = [ 1, 7, @@ -83,18 +85,61 @@ const badPorts = [ 10080 ] -module.exports = { - badPort (request) { - // 1. Let url be request’s current URL. - const url = request.currentURL +function responseURL (response) { + // https://fetch.spec.whatwg.org/#responses + // A response has an associated URL. It is a pointer to the last URL + // in response’s URL list and null if response’s URL list is empty. + const urlList = response.urlList + const length = urlList.length + return length === 0 ? null : urlList[length - 1].toString() +} + +// https://fetch.spec.whatwg.org/#concept-response-location-url +function responseLocationURL (response, requestFragment) { + // 1. If response’s status is not a redirect status, then return null. + if (!redirectStatus.includes(response.status)) { + return null + } - // 2. If url’s scheme is an HTTP(S) scheme and url’s port is a bad port, - // then return blocked. - if (/^http?s/.test(url.protocol) && badPorts.includes(url.port)) { - return 'blocked' - } + // 2. Let location be the result of extracting header list values given + // `Location` and response’s header list. + let location = response.headersList.get('location') - // 3. Return allowed. - return 'allowed' + // 3. If location is a value, then set location to the result of parsing + // location with response’s URL. + location = location ? new URL(location, responseURL(response)) : null + + // 4. If location is a URL whose fragment is null, then set location’s + // fragment to requestFragment. + if (location && !location.hash) { + location.hash = requestFragment } + + // 5. Return location. + return location +} + +function requestCurrentURL (request) { + return request.urlList[request.urlList.length - 1] +} + +function requestBadPort (request) { + // 1. Let url be request’s current URL. + const url = requestCurrentURL(request) + + // 2. If url’s scheme is an HTTP(S) scheme and url’s port is a bad port, + // then return blocked. + if (/^http?s/.test(url.protocol) && badPorts.includes(url.port)) { + return 'blocked' + } + + // 3. Return allowed. + return 'allowed' +} + +module.exports = { + requestBadPort, + requestCurrentURL, + responseURL, + responseLocationURL } diff --git a/lib/client.js b/lib/client.js index d042784969b..8d3ce51be39 100644 --- a/lib/client.js +++ b/lib/client.js @@ -406,7 +406,10 @@ const llhttp = new WebAssembly.Instance(mod, { return 0 }, wasm_on_status: (p, at, len) => { - return 0 + assert.strictEqual(currentParser.ptr, p) + const start = at - currentBufferPtr + const end = start + len + return currentParser.onStatus(currentBufferRef.slice(start, end)) || 0 }, wasm_on_message_begin: (p) => { assert.strictEqual(currentParser.ptr, p) @@ -463,6 +466,7 @@ class Parser { this.timeoutValue = null this.timeoutType = null this.statusCode = null + this.statusText = '' this.upgrade = false this.headers = [] this.headersSize = 0 @@ -653,6 +657,10 @@ class Parser { this.paused = false } + onStatus (buf) { + this.statusText = buf.toString() + } + onMessageBegin () { const { socket, client } = this @@ -722,6 +730,7 @@ class Parser { assert(request.upgrade || request.method === 'CONNECT') this.statusCode = null + this.statusText = '' this.shouldKeepAlive = null assert(this.headers.length % 2 === 0) @@ -755,7 +764,7 @@ class Parser { } onHeadersComplete (statusCode, upgrade, shouldKeepAlive) { - const { client, socket, headers } = this + const { client, socket, headers, statusText } = this /* istanbul ignore next: difficult to make a test case for */ if (socket.destroyed) { @@ -842,7 +851,7 @@ class Parser { } try { - if (request.onHeaders(statusCode, headers, this.resume) === false) { + if (request.onHeaders(statusCode, headers, this.resume, statusText) === false) { return constants.ERROR.PAUSED } } catch (err) { @@ -909,6 +918,7 @@ class Parser { assert(statusCode >= 100) this.statusCode = null + this.statusText = '' this.bytesRead = 0 this.contentLength = '' this.trailer = '' diff --git a/lib/core/request.js b/lib/core/request.js index f242930c1b9..5e3278c60ac 100644 --- a/lib/core/request.js +++ b/lib/core/request.js @@ -133,11 +133,11 @@ class Request { return this[kHandler].onConnect(abort) } - onHeaders (statusCode, headers, resume) { + onHeaders (statusCode, headers, resume, statusText) { assert(!this.aborted) assert(!this.completed) - return this[kHandler].onHeaders(statusCode, headers, resume) + return this[kHandler].onHeaders(statusCode, headers, resume, statusText) } onData (chunk) { diff --git a/lib/handler/redirect.js b/lib/handler/redirect.js index 46b0ea8900a..776fd082a48 100644 --- a/lib/handler/redirect.js +++ b/lib/handler/redirect.js @@ -86,13 +86,13 @@ class RedirectHandler { this.handler.onError(error) } - onHeaders (statusCode, headers, resume) { + onHeaders (statusCode, headers, resume, statusText) { this.location = this.history.length >= this.maxRedirections || util.isDisturbed(this.opts.body) ? null : parseLocation(statusCode, headers) if (!this.location) { - return this.handler.onHeaders(statusCode, headers, resume) + return this.handler.onHeaders(statusCode, headers, resume, statusText) } this.history.push(new URL(this.opts.path, this.opts.origin)) diff --git a/test/node-fetch/headers.js b/test/node-fetch/headers.js index 44f9bcc8f74..0a9212cf85a 100644 --- a/test/node-fetch/headers.js +++ b/test/node-fetch/headers.js @@ -233,7 +233,7 @@ describe('Headers', () => { expect(h3Raw.b).to.include('1') }) - xit('should accept headers as an iterable of tuples', () => { + it('should accept headers as an iterable of tuples', () => { let headers headers = new Headers([ @@ -264,7 +264,7 @@ describe('Headers', () => { expect(() => new Headers([['b', '2', 'huh?']])).to.throw(TypeError) expect(() => new Headers(['b2'])).to.throw(TypeError) expect(() => new Headers('b2')).to.throw(TypeError) - // expect(() => new Headers({ [Symbol.iterator]: 42 })).to.throw(TypeError) + expect(() => new Headers({ [Symbol.iterator]: 42 })).to.throw(TypeError) }) xit('should use a custom inspect function', () => { diff --git a/test/node-fetch/main.js b/test/node-fetch/main.js index d64353ce11a..fc6ebd37f64 100644 --- a/test/node-fetch/main.js +++ b/test/node-fetch/main.js @@ -534,7 +534,7 @@ describe('node-fetch', () => { }) }) - xit('should handle response with no status text', () => { + it('should handle response with no status text', () => { const url = `${base}no-status-text` return fetch(url).then(res => { expect(res.statusText).to.equal('') @@ -894,36 +894,6 @@ describe('node-fetch', () => { }) }) - xit('should cancel request body of type Stream with AbortError when aborted', () => { - const body = toWeb(new stream.Readable({ objectMode: true })) - body._read = () => {} - const promise = fetch( - `${base}slow`, - { signal: controller.signal, body, method: 'POST' } - ) - - const result = Promise.all([ - new Promise((resolve, reject) => { - body.on('error', error => { - try { - expect(error).to.be.an.instanceof(Error) - .and.have.property('name', 'AbortError') - resolve() - } catch (error_) { - reject(error_) - } - }) - }), - expect(promise).to.eventually.be.rejected - .and.be.an.instanceof(Error) - .and.have.property('name', 'AbortError') - ]) - - controller.abort() - - return result - }) - if (moreTests) { moreTests() } @@ -1376,7 +1346,7 @@ describe('node-fetch', () => { }) }) - it('should allow cloning a json response and log it as text response', () => { + xit('should allow cloning a json response and log it as text response', () => { const url = `${base}json` return fetch(url).then(res => { const r1 = res.clone() @@ -1387,7 +1357,7 @@ describe('node-fetch', () => { }) }) - it('should allow cloning a json response, and then log it as text response', () => { + xit('should allow cloning a json response, and then log it as text response', () => { const url = `${base}json` return fetch(url).then(res => { const r1 = res.clone() @@ -1400,7 +1370,7 @@ describe('node-fetch', () => { }) }) - it('should allow cloning a json response, first log as text response, then return json object', () => { + xit('should allow cloning a json response, first log as text response, then return json object', () => { const url = `${base}json` return fetch(url).then(res => { const r1 = res.clone() From ffeba7b7a37262dbe41f9fcae306e802fcef0167 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Wed, 11 Aug 2021 12:00:11 +0200 Subject: [PATCH 08/56] fixup: isBodyReadable --- lib/api/api-fetch/body.js | 7 ++ lib/api/api-fetch/index.js | 28 ++--- test/node-fetch/main.js | 240 ++++++++++++++++++------------------- 3 files changed, 134 insertions(+), 141 deletions(-) diff --git a/lib/api/api-fetch/body.js b/lib/api/api-fetch/body.js index 566d0530370..18745ea78a7 100644 --- a/lib/api/api-fetch/body.js +++ b/lib/api/api-fetch/body.js @@ -7,6 +7,7 @@ const { NotSupportedError } = require('../../core/errors') const { ReadableStream } = require('stream/web') const { kBodyUsed } = require('../../core/symbols') const assert = require('assert') +const nodeUtil = require('util') // https://fetch.spec.whatwg.org/#concept-bodyinit-extract function extractBody (object, keepalive = false) { @@ -254,8 +255,14 @@ function mixinBody (prototype) { Object.defineProperties(prototype, properties) } +function isBodyReadable (body) { + // This is a hack! + return body && /state: 'readable'/.test(nodeUtil.inspect(body.stream)) +} + module.exports = { extractBody, + isBodyReadable, safelyExtractBody, cloneBody, mixinBody diff --git a/lib/api/api-fetch/index.js b/lib/api/api-fetch/index.js index b4769a9dcd7..d0e4b568550 100644 --- a/lib/api/api-fetch/index.js +++ b/lib/api/api-fetch/index.js @@ -9,7 +9,7 @@ const { requestBadPort, responseLocationURL, requestCurrentURL } = require('./ut const { kState, kHeaders, kGuard } = require('./symbols') const { AbortError } = require('../../core/errors') const assert = require('assert') -const { safelyExtractBody } = require('./body') +const { safelyExtractBody, isBodyReadable } = require('./body') const { redirectStatus, nullBodyStatus, @@ -57,7 +57,7 @@ async function fetch (resource, init) { // 4. If requestObject’s signal’s aborted flag is set, then: if (requestObject.signal.aborted) { // 1. Abort fetch with p, request, and null. - abortFetch(p, request, null) + abortFetch.call(context, p, request, null) // 2. Return p. return p.promise @@ -85,7 +85,7 @@ async function fetch (resource, init) { locallyAborted = true // 2. Abort fetch with p, request, and responseObject. - abortFetch(p, request, responseObject) + abortFetch.call(context, p, request, responseObject) // 3. Terminate the ongoing fetch with the aborted flag set. context.terminate({ aborted: true }) @@ -106,7 +106,7 @@ async function fetch (resource, init) { // 2. If response’s aborted flag is set, then abort fetch with p, // request, and responseObject, and terminate these substeps. if (response.aborted) { - abortFetch(p, request, responseObject) + abortFetch.call(context, p, request, responseObject) return } @@ -147,6 +147,8 @@ function finalizeAndReportTiming (response, initiatorType = 'other') { // https://fetch.spec.whatwg.org/#abort-fetch function abortFetch (p, request, responseObject) { + const context = this + // 1. Let error be an "AbortError" DOMException. const error = new AbortError() @@ -155,18 +157,8 @@ function abortFetch (p, request, responseObject) { // 3. If request’s body is not null and is readable, then cancel request’s // body with error. - if (request.body !== null) { - // NOTE: We don't have any way to check if body is readable. However, - // cancel will be noop if state is "closed" and reject if state is - // "error", which means that calling cancel() and catch errors - // has the same effect as checking for "readable". - - try { - // TODO: What if locked? - request.body.stream.cancel(error) - } catch { - // Do nothing... - } + if (request.body !== null && isBodyReadable(request.body)) { + request.body.stream.cancel(error) } // 4. If responseObject is null, then return. @@ -175,9 +167,11 @@ function abortFetch (p, request, responseObject) { } // 5. Let response be responseObject’s response. + const response = responseObject[kState] + // 6. If response’s body is not null and is readable, then error response’s // body with error. - if (context.controller) { + if (response.body != null && isBodyReadable(response.body)) { context.controller.error(error) } } diff --git a/test/node-fetch/main.js b/test/node-fetch/main.js index fc6ebd37f64..38588ccbfb7 100644 --- a/test/node-fetch/main.js +++ b/test/node-fetch/main.js @@ -766,141 +766,133 @@ describe('node-fetch', () => { }) }) - const testAbortController = (name, buildAbortController, moreTests = null) => { - describe(`AbortController (${name})`, () => { - let controller - - beforeEach(() => { - controller = buildAbortController() - }) - - it('should support request cancellation with signal', () => { - const fetches = [ - fetch( - `${base}timeout`, - { - method: 'POST', - signal: controller.signal, - headers: { - 'Content-Type': 'application/json', - body: JSON.stringify({ hello: 'world' }) - } + describe('AbortController', () => { + let controller + + beforeEach(() => { + controller = new AbortController() + }) + + it('should support request cancellation with signal', () => { + const fetches = [ + fetch( + `${base}timeout`, + { + method: 'POST', + signal: controller.signal, + headers: { + 'Content-Type': 'application/json', + body: JSON.stringify({ hello: 'world' }) } - ) - ] - setTimeout(() => { - controller.abort() - }, 100) - - return Promise.all(fetches.map(fetched => expect(fetched) - .to.eventually.be.rejected - .and.be.an.instanceOf(Error) - .and.have.property('name', 'AbortError') - )) - }) - - it('should support multiple request cancellation with signal', () => { - const fetches = [ - fetch(`${base}timeout`, { signal: controller.signal }), - fetch( - `${base}timeout`, - { - method: 'POST', - signal: controller.signal, - headers: { - 'Content-Type': 'application/json', - body: JSON.stringify({ hello: 'world' }) - } + } + ) + ] + setTimeout(() => { + controller.abort() + }, 100) + + return Promise.all(fetches.map(fetched => expect(fetched) + .to.eventually.be.rejected + .and.be.an.instanceOf(Error) + .and.have.property('name', 'AbortError') + )) + }) + + it('should support multiple request cancellation with signal', () => { + const fetches = [ + fetch(`${base}timeout`, { signal: controller.signal }), + fetch( + `${base}timeout`, + { + method: 'POST', + signal: controller.signal, + headers: { + 'Content-Type': 'application/json', + body: JSON.stringify({ hello: 'world' }) } - ) - ] - setTimeout(() => { - controller.abort() - }, 100) - - return Promise.all(fetches.map(fetched => expect(fetched) - .to.eventually.be.rejected - .and.be.an.instanceOf(Error) - .and.have.property('name', 'AbortError') - )) - }) - - it('should reject immediately if signal has already been aborted', () => { - const url = `${base}timeout` - const options = { - signal: controller.signal - } + } + ) + ] + setTimeout(() => { controller.abort() - const fetched = fetch(url, options) - return expect(fetched).to.eventually.be.rejected - .and.be.an.instanceOf(Error) - .and.have.property('name', 'AbortError') - }) + }, 100) - it('should allow redirects to be aborted', () => { - const request = new Request(`${base}redirect/slow`, { - signal: controller.signal - }) - setTimeout(() => { - controller.abort() - }, 20) - return expect(fetch(request)).to.be.eventually.rejected - .and.be.an.instanceOf(Error) - .and.have.property('name', 'AbortError') - }) + return Promise.all(fetches.map(fetched => expect(fetched) + .to.eventually.be.rejected + .and.be.an.instanceOf(Error) + .and.have.property('name', 'AbortError') + )) + }) - it('should allow redirected response body to be aborted', () => { - const request = new Request(`${base}redirect/slow-stream`, { - signal: controller.signal - }) - return expect(fetch(request).then(res => { - expect(res.headers.get('content-type')).to.equal('text/plain') - const result = res.text() - controller.abort() - return result - })).to.be.eventually.rejected - .and.be.an.instanceOf(Error) - .and.have.property('name', 'AbortError') - }) + it('should reject immediately if signal has already been aborted', () => { + const url = `${base}timeout` + const options = { + signal: controller.signal + } + controller.abort() + const fetched = fetch(url, options) + return expect(fetched).to.eventually.be.rejected + .and.be.an.instanceOf(Error) + .and.have.property('name', 'AbortError') + }) - it('should reject response body with AbortError when aborted before stream has been read completely', () => { - return expect(fetch( - `${base}slow`, - { signal: controller.signal } - )) - .to.eventually.be.fulfilled - .then(res => { - const promise = res.text() - controller.abort() - return expect(promise) - .to.eventually.be.rejected - .and.be.an.instanceof(Error) - .and.have.property('name', 'AbortError') - }) + it('should allow redirects to be aborted', () => { + const request = new Request(`${base}redirect/slow`, { + signal: controller.signal }) + setTimeout(() => { + controller.abort() + }, 20) + return expect(fetch(request)).to.be.eventually.rejected + .and.be.an.instanceOf(Error) + .and.have.property('name', 'AbortError') + }) - it('should reject response body methods immediately with AbortError when aborted before stream is disturbed', () => { - return expect(fetch( - `${base}slow`, - { signal: controller.signal } - )) - .to.eventually.be.fulfilled - .then(res => { - controller.abort() - return expect(res.text()) - .to.eventually.be.rejected - .and.be.an.instanceof(Error) - .and.have.property('name', 'AbortError') - }) + it('should allow redirected response body to be aborted', () => { + const request = new Request(`${base}redirect/slow-stream`, { + signal: controller.signal }) - - if (moreTests) { - moreTests() - } + return expect(fetch(request).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain') + const result = res.text() + controller.abort() + return result + })).to.be.eventually.rejected + .and.be.an.instanceOf(Error) + .and.have.property('name', 'AbortError') + }) + + it('should reject response body with AbortError when aborted before stream has been read completely', () => { + return expect(fetch( + `${base}slow`, + { signal: controller.signal } + )) + .to.eventually.be.fulfilled + .then(res => { + const promise = res.text() + controller.abort() + return expect(promise) + .to.eventually.be.rejected + .and.be.an.instanceof(Error) + .and.have.property('name', 'AbortError') + }) }) - } - testAbortController('native', () => new AbortController()) + it('should reject response body methods immediately with AbortError when aborted before stream is disturbed', () => { + return expect(fetch( + `${base}slow`, + { signal: controller.signal } + )) + .to.eventually.be.fulfilled + .then(res => { + controller.abort() + return expect(res.text()) + .to.eventually.be.rejected + .and.be.an.instanceof(Error) + .and.have.property('name', 'AbortError') + }) + }) + }) it('should throw a TypeError if a signal is not of type AbortSignal or EventTarget', () => { return Promise.all([ From 7c288ea4ad74f641e521fad1b8dfe4972db33f93 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Wed, 11 Aug 2021 12:12:53 +0200 Subject: [PATCH 09/56] fixup: localURLSOnly --- lib/api/api-fetch/index.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/api/api-fetch/index.js b/lib/api/api-fetch/index.js index d0e4b568550..76242aef81e 100644 --- a/lib/api/api-fetch/index.js +++ b/lib/api/api-fetch/index.js @@ -285,7 +285,9 @@ async function mainFetch (fetchParams, recursive = false) { // 3. If request’s local-URLs-only flag is set and request’s current URL is // not local, then set response to a network error. - // TODO + if (request.localURLsOnly && !/^(about|blob|data):/.test(requestCurrentURL(request).protocol)) { + return makeNetworkError('local URLs only') + } // 4. Run report Content Security Policy violations for request. // TODO From c03a784e7d1ec10df1c3b3834aa790f319fd8801 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Wed, 11 Aug 2021 12:16:12 +0200 Subject: [PATCH 10/56] fixup: timingAllow --- lib/api/api-fetch/index.js | 4 +++- lib/api/api-fetch/response.js | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/api/api-fetch/index.js b/lib/api/api-fetch/index.js index 76242aef81e..001851373d3 100644 --- a/lib/api/api-fetch/index.js +++ b/lib/api/api-fetch/index.js @@ -446,7 +446,9 @@ async function mainFetch (fetchParams, recursive = false) { // 16. If request’s timing allow failed flag is unset, then set // internalResponse’s timing allow passed flag. - // TODO + if (!request.timingAllowFailed) { + response.timingAllowPassed = true + } // 17. If response is not a network error and any of the following returns // blocked diff --git a/lib/api/api-fetch/response.js b/lib/api/api-fetch/response.js index 7eef9e0f98e..564a2aef703 100644 --- a/lib/api/api-fetch/response.js +++ b/lib/api/api-fetch/response.js @@ -275,6 +275,7 @@ function makeResponse (init) { internalResponse: null, aborted: false, rangeRequested: false, + timingAllowPassed: false, type: 'default', status: 200, statusText: '', From c6114934bd7df878f1265462a708b8206f322574 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Wed, 11 Aug 2021 12:19:55 +0200 Subject: [PATCH 11/56] fixup: timingInfo --- lib/api/api-fetch/index.js | 7 +++++-- lib/api/api-fetch/response.js | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/api/api-fetch/index.js b/lib/api/api-fetch/index.js index 001851373d3..0813b755c2a 100644 --- a/lib/api/api-fetch/index.js +++ b/lib/api/api-fetch/index.js @@ -193,10 +193,12 @@ function fetching ({ // 4. If useParallelQueue is true, then set taskDestination to the result of // starting a new parallel queue. + // TODO // 5. Let timingInfo be a new fetch timing info whose start time and // post-redirect start time are the coarsened shared current time given // crossOriginIsolatedCapability. + // TODO // 6. Let fetchParams be a new fetch params whose request is request, timing // info is timingInfo, process request body is processRequestBody, @@ -207,6 +209,7 @@ function fetching ({ // is crossOriginIsolatedCapability. const fetchParams = { request, + timingInfo: null, processRequestBody: null, processRequestEndOfBody: null, processResponse, @@ -579,7 +582,7 @@ async function httpFetch (fetchParams) { let actualResponse = null // 4. Let timingInfo be fetchParams’s timing info. - // TODO + const timingInfo = fetchParams.timingInfo // 5. If request’s service-workers mode is "all", then: // TODO @@ -642,7 +645,7 @@ async function httpFetch (fetchParams) { } // 9. Set response’s timing info to timingInfo. - // TODO + response.timingInfo = timingInfo // 10. Return response. return response diff --git a/lib/api/api-fetch/response.js b/lib/api/api-fetch/response.js index 564a2aef703..2b4dceaef0b 100644 --- a/lib/api/api-fetch/response.js +++ b/lib/api/api-fetch/response.js @@ -278,6 +278,7 @@ function makeResponse (init) { timingAllowPassed: false, type: 'default', status: 200, + timingInfo: null, statusText: '', ...init, headersList: init.headersList From 360530b46e02c20576cc279654893c6af96ec24e Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Wed, 11 Aug 2021 12:29:52 +0200 Subject: [PATCH 12/56] fixup: disregard any enqueuing --- lib/api/api-fetch/index.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/api/api-fetch/index.js b/lib/api/api-fetch/index.js index 0813b755c2a..e446274ea27 100644 --- a/lib/api/api-fetch/index.js +++ b/lib/api/api-fetch/index.js @@ -487,6 +487,9 @@ async function mainFetch (fetchParams, recursive = false) { ) ) { internalResponse.body = null + if (context.abort) { + context.abort() + } } // 20. If request’s integrity metadata is not the empty string, then: From 9c9fa53d57e5f7b7c1332a47bcec6efecece88f4 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Wed, 11 Aug 2021 12:31:06 +0200 Subject: [PATCH 13/56] fixup: strict cmp --- lib/api/api-fetch/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/api/api-fetch/index.js b/lib/api/api-fetch/index.js index e446274ea27..94920e83407 100644 --- a/lib/api/api-fetch/index.js +++ b/lib/api/api-fetch/index.js @@ -309,7 +309,7 @@ async function mainFetch (fetchParams, recursive = false) { // 7. If request’s referrer policy is the empty string, then set request’s // referrer policy to request’s policy container’s referrer policy. - if (!request.referrerPolicy) { + if (request.referrerPolicy === '') { // TODO } From 9172bec7d9eefbfd7032a85df3cff9ad584906bb Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Wed, 11 Aug 2021 12:44:22 +0200 Subject: [PATCH 14/56] fixup: clone --- lib/api/api-fetch/response.js | 5 +---- test/node-fetch/main.js | 6 +++--- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/lib/api/api-fetch/response.js b/lib/api/api-fetch/response.js index 2b4dceaef0b..894dd5f4fe2 100644 --- a/lib/api/api-fetch/response.js +++ b/lib/api/api-fetch/response.js @@ -251,10 +251,7 @@ function cloneResponse (response) { // filtered response whose internal response is a clone of response’s // internal response. if (response.internalResponse) { - return { - ...response, - internalResponse: cloneResponse(response.internalResponse) - } + return filterResponse(cloneResponse(response.internalResponse), response.type) } // 2. Let newResponse be a copy of response, except for its body. diff --git a/test/node-fetch/main.js b/test/node-fetch/main.js index 38588ccbfb7..ed5e36e11d4 100644 --- a/test/node-fetch/main.js +++ b/test/node-fetch/main.js @@ -1338,7 +1338,7 @@ describe('node-fetch', () => { }) }) - xit('should allow cloning a json response and log it as text response', () => { + it('should allow cloning a json response and log it as text response', () => { const url = `${base}json` return fetch(url).then(res => { const r1 = res.clone() @@ -1349,7 +1349,7 @@ describe('node-fetch', () => { }) }) - xit('should allow cloning a json response, and then log it as text response', () => { + it('should allow cloning a json response, and then log it as text response', () => { const url = `${base}json` return fetch(url).then(res => { const r1 = res.clone() @@ -1362,7 +1362,7 @@ describe('node-fetch', () => { }) }) - xit('should allow cloning a json response, first log as text response, then return json object', () => { + it('should allow cloning a json response, first log as text response, then return json object', () => { const url = `${base}json` return fetch(url).then(res => { const r1 = res.clone() From 362ef3355b9a0183ae8a5991b97da962689514e8 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Wed, 11 Aug 2021 12:45:53 +0200 Subject: [PATCH 15/56] fixup: don't need to copy URL --- lib/api/api-fetch/response.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/api/api-fetch/response.js b/lib/api/api-fetch/response.js index 894dd5f4fe2..9a91d301787 100644 --- a/lib/api/api-fetch/response.js +++ b/lib/api/api-fetch/response.js @@ -282,7 +282,7 @@ function makeResponse (init) { ? new HeadersList(...init.headersList) : new HeadersList(), urlList: init.urlList - ? [...init.urlList.map(url => new URL(url))] + ? [...init.urlList] : [] } } From 1c42102f59de0647565099b841563e50cafce5ad Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Wed, 11 Aug 2021 15:09:38 +0200 Subject: [PATCH 16/56] fix: Response init --- lib/api/api-fetch/response.js | 2 +- test/node-fetch/response.js | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/api/api-fetch/response.js b/lib/api/api-fetch/response.js index 9a91d301787..697bdbe13e5 100644 --- a/lib/api/api-fetch/response.js +++ b/lib/api/api-fetch/response.js @@ -96,7 +96,7 @@ class Response { } // 3. Set this’s response to a new response. - this[kState] = makeResponse(init) + this[kState] = makeResponse({}) // 4. Set this’s headers to a new Headers object with this’s relevant // Realm, whose header list is this’s response’s header list and guard diff --git a/test/node-fetch/response.js b/test/node-fetch/response.js index 62417f4b7ad..42e477f1b73 100644 --- a/test/node-fetch/response.js +++ b/test/node-fetch/response.js @@ -6,7 +6,7 @@ const { Response } = require('../../lib/api/api-fetch/response.js') const TestServer = require('./utils/server.js') const toWeb = require('./utils/toWeb.js') const { Blob } = require('buffer') -const { kUrlList } = require('../../lib/api/api-fetch/symbols.js') +const { kState } = require('../../lib/api/api-fetch/symbols.js') const { expect } = chai @@ -114,15 +114,15 @@ describe('Response', () => { } xit('should support clone() method', () => { - const body = stream.Readable.from('a=1') + const body = toWeb(stream.Readable.from('a=1')) const res = new Response(body, { headers: { a: '1' }, - [kUrlList]: base, status: 346, statusText: 'production' }) + res[kState].urlList = [base] const cl = res.clone() expect(cl.headers.get('a')).to.equal('1') expect(cl.type).to.equal('default') From 883a56cdbaf8ed71c3e082183b0a76cd2dc9feb3 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Wed, 11 Aug 2021 15:10:07 +0200 Subject: [PATCH 17/56] fixup: opaqueredirect --- lib/api/api-fetch/response.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/api/api-fetch/response.js b/lib/api/api-fetch/response.js index 697bdbe13e5..9e930170e15 100644 --- a/lib/api/api-fetch/response.js +++ b/lib/api/api-fetch/response.js @@ -353,7 +353,7 @@ function filterResponse (response, type) { return makeResponse({ ...response, internalResponse: response, - type: 'opaque', + type: 'opaqueredirect', status: 0, statusText: '', headersList: new HeadersList(), From 4724d369bc49bb51b58e9e3d27f6175ae67d8c08 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Wed, 11 Aug 2021 15:27:05 +0200 Subject: [PATCH 18/56] fixup: avoid new Response --- lib/api/api-fetch/index.js | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/api/api-fetch/index.js b/lib/api/api-fetch/index.js index 94920e83407..cac542fdd44 100644 --- a/lib/api/api-fetch/index.js +++ b/lib/api/api-fetch/index.js @@ -2,7 +2,7 @@ 'use strict' -const { Response, makeNetworkError, filterResponse } = require('./response') +const { Response, makeNetworkError, filterResponse, makeResponse } = require('./response') const { Headers } = require('./headers') const { Request, makeRequest } = require('./request') const { requestBadPort, responseLocationURL, requestCurrentURL } = require('./util') @@ -1188,9 +1188,13 @@ function httpNetworkFetch ( } }, { highWaterMark: 16384 }) - // TODO: Not sure we should use new Response here? - response = new Response(stream, { status, statusText })[kState] - response.headersList.push(...headers[kHeadersList]) + response = makeResponse({ + status, + statusText, + headersList: + headers[kHeadersList], + body: stream ? { stream } : null + }) resolve(response) @@ -1221,7 +1225,7 @@ function httpNetworkFetch ( onError (err) { if (context.controller) { - context.controller?.error(err) + context.controller.error(err) context.controller = null context.abort = null } else { From 86474709d9d43a568a4c5e88a5187c2fb59a6c8b Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Wed, 11 Aug 2021 15:38:48 +0200 Subject: [PATCH 19/56] fixup: abort default err --- lib/api/api-fetch/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/api/api-fetch/index.js b/lib/api/api-fetch/index.js index cac542fdd44..ecb61253a1e 100644 --- a/lib/api/api-fetch/index.js +++ b/lib/api/api-fetch/index.js @@ -1147,7 +1147,7 @@ function httpNetworkFetch ( if (context.terminated) { abort(new AbortError()) } else { - context.abort = () => abort(new AbortError()) + context.abort = (err) => abort(err ?? new AbortError()) } }, From db693fc03c7b56c3ed85786abacd1e7447a34dba Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Wed, 11 Aug 2021 15:39:34 +0200 Subject: [PATCH 20/56] fixup: url --- lib/api/api-fetch/index.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/api/api-fetch/index.js b/lib/api/api-fetch/index.js index ecb61253a1e..ea7a5006ea6 100644 --- a/lib/api/api-fetch/index.js +++ b/lib/api/api-fetch/index.js @@ -1135,9 +1135,10 @@ function httpNetworkFetch ( assert(!context.controller) + const url = requestCurrentURL(request) return new Promise((resolve) => context.dispatcher.dispatch({ - path: requestCurrentURL(request).pathname + requestCurrentURL(request).search, - origin: requestCurrentURL(request).origin, + path: url.pathname + url.search, + origin: url.origin, method: request.method, body, headers: request.headersList, From 6c0d4bd6bf309a45a16d3d592382aed5e945e5d9 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Wed, 11 Aug 2021 15:48:04 +0200 Subject: [PATCH 21/56] fixup: don't use body.source when writing request body --- lib/api/api-fetch/body.js | 7 ++++--- lib/api/api-fetch/index.js | 18 +----------------- 2 files changed, 5 insertions(+), 20 deletions(-) diff --git a/lib/api/api-fetch/body.js b/lib/api/api-fetch/body.js index 18745ea78a7..8c4e9667d5a 100644 --- a/lib/api/api-fetch/body.js +++ b/lib/api/api-fetch/body.js @@ -62,9 +62,10 @@ function extractBody (object, keepalive = false) { } // Set source to a copy of the bytes held by object. - source = util.isBuffer(object) - ? Buffer.from(object) // Buffer.slice references same memory. - : object.slice(0) + source = new Uint8Array(object) + + // TODO: This is not part of spec. + length = source.byteLength } else if (object instanceof Blob) { // Blob diff --git a/lib/api/api-fetch/index.js b/lib/api/api-fetch/index.js index ea7a5006ea6..2ad63eb9e4b 100644 --- a/lib/api/api-fetch/index.js +++ b/lib/api/api-fetch/index.js @@ -1108,22 +1108,6 @@ function httpNetworkFetch ( // ... - let body = null - if (request.body) { - // TODO (fix): Do we need to lock the Request stream and if - // so do we need to do it immediatly? - - if (request.body.source) { - // We can bypass stream and use source directly - // since Request already should have locked the - // source stream thus making it "unusable" for - // anyone else. - body = request.body.source - } else { - body = request.body.stream - } - } - const context = this // TODO: forceNewConnection @@ -1140,7 +1124,7 @@ function httpNetworkFetch ( path: url.pathname + url.search, origin: url.origin, method: request.method, - body, + body: request.body ? request.body.stream : request.body, headers: request.headersList, maxRedirections: 0 }, { From f81577a6acd972a13ea954cb666657f5e6e30948 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Wed, 11 Aug 2021 17:28:38 +0200 Subject: [PATCH 22/56] fixup --- lib/api/api-fetch/index.js | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/lib/api/api-fetch/index.js b/lib/api/api-fetch/index.js index 2ad63eb9e4b..f8197ea9090 100644 --- a/lib/api/api-fetch/index.js +++ b/lib/api/api-fetch/index.js @@ -882,6 +882,29 @@ async function httpNetworkOrCacheFetch ( // TODO // 12. Append the Fetch metadata headers for httpRequest. [FETCH-METADATA] + + // https://w3c.github.io/webappsec-fetch-metadata/#sec-fetch-dest-header + // TODO + + // https://w3c.github.io/webappsec-fetch-metadata/#sec-fetch-mode-header + { + // 1. Assert: r’s url is a potentially trustworthy URL. + // TODO + + // 2. Let header be a Structured Header whose value is a token. + let header = null + + // 3. Set header’s value to r’s mode. + header = request.mode + + // 4. Set a structured field value `Sec-Fetch-Mode`/header in r’s header list. + httpRequest.headersList.append('sec-fetch-mode', header) + } + + // https://w3c.github.io/webappsec-fetch-metadata/#sec-fetch-site-header + // TODO + + // https://w3c.github.io/webappsec-fetch-metadata/#sec-fetch-user-header // TODO // 13. If httpRequest’s header list does not contain `User-Agent`, then From 515d6c1745d6ad09103a725a2630b6de0df925e7 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Wed, 11 Aug 2021 17:44:25 +0200 Subject: [PATCH 23/56] fixup --- lib/api/api-fetch/index.js | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/lib/api/api-fetch/index.js b/lib/api/api-fetch/index.js index f8197ea9090..5a8aba53ccc 100644 --- a/lib/api/api-fetch/index.js +++ b/lib/api/api-fetch/index.js @@ -1058,6 +1058,28 @@ async function httpNetworkOrCacheFetch ( // 14. If response’s status is 407, then: if (response.status === 407) { + // 1. If request’s window is "no-window", then return a network error. + if (request.window === 'no-window') { + return makeNetworkError() + } + + // 2. ??? + + // 3. If the ongoing fetch is terminated, then: + if (context.terminated) { + // 1. Let aborted be the termination’s aborted flag. + // 2. If aborted is set, then return an aborted network error. + // 3. Return a network error. + return makeNetworkError(context.terminated.aborted ? new AbortError() : null) + } + + // 4. Prompt the end user as appropriate in request’s window and store + // the result as a proxy-authentication entry. [HTTP-AUTH] + // TODO: Invoke some kind of callback? + return makeNetworkError('proxy authentication required') + + // 5. Set response to the result of running HTTP-network-or-cache fetch given + // fetchParams. // TODO } From e475f5e492375800c9947f1b5e95d957037cf0f9 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 08:19:09 +0200 Subject: [PATCH 24/56] fixup: fixes from review --- lib/api/api-fetch/body.js | 10 +++++++--- lib/api/api-fetch/response.js | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/api/api-fetch/body.js b/lib/api/api-fetch/body.js index 8c4e9667d5a..046bbe743fe 100644 --- a/lib/api/api-fetch/body.js +++ b/lib/api/api-fetch/body.js @@ -40,7 +40,10 @@ function extractBody (object, keepalive = false) { let contentType = null // 6. Switch on object: - if (object instanceof URLSearchParams) { + if (object == null) { + // Note: The IDL processor cannot handle this situation. See + // https://crbug.com/335871. + } else if (object instanceof URLSearchParams) { // URLSearchParams // spec says to run application/x-www-form-urlencoded on body.list @@ -132,8 +135,9 @@ function extractBody (object, keepalive = false) { // Whenever one or more bytes are available and stream is not errored, // enqueue a Uint8Array wrapping an ArrayBuffer containing the available // bytes into stream. - // TODO: stream is not errored? - controller.enqueue(new Uint8Array(bytes)) + if (!/state: 'errored'/.test(nodeUtil.inspect(stream))) { + controller.enqueue(new Uint8Array(bytes)) + } }, err => { // TODO: Spec doesn't say anything about this? controller.error(err) diff --git a/lib/api/api-fetch/response.js b/lib/api/api-fetch/response.js index 9e930170e15..268d5aede9a 100644 --- a/lib/api/api-fetch/response.js +++ b/lib/api/api-fetch/response.js @@ -46,7 +46,7 @@ class Response { // 3. If status is not a redirect status, then throw a RangeError. if (!redirectStatus.includes(status)) { - throw new RangeError() + throw new RangeError(`Failed to construct 'Response': The status provided (${status}) is outside the range [200, 599].`) } // 4. Let responseObject be the result of creating a Response object, From 866e64bf81089ce1023a648cc461ccb5733219d9 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 08:25:55 +0200 Subject: [PATCH 25/56] fixup: check valid reason phrase --- lib/api/api-fetch/response.js | 8 +++++--- lib/api/api-fetch/util.js | 23 ++++++++++++++++++++++- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/lib/api/api-fetch/response.js b/lib/api/api-fetch/response.js index 268d5aede9a..efad93c127b 100644 --- a/lib/api/api-fetch/response.js +++ b/lib/api/api-fetch/response.js @@ -4,7 +4,7 @@ const { Headers, HeadersList, fill } = require('./headers') const { extractBody, cloneBody, mixinBody } = require('./body') const util = require('../../core/util') const { kEnumerableProperty } = util -const { responseURL } = require('./util') +const { responseURL, isValidReasonPhrase } = require('./util') const { redirectStatus, nullBodyStatus, forbiddenHeaderNames } = require('./constants') const assert = require('assert') const { @@ -85,14 +85,16 @@ class Response { if ('statusText' in init) { if (typeof init.statusText !== 'string') { - throw new TypeError() + throw new TypeError('Invalid statusText') } // 2. If init["statusText"] does not match the reason-phrase token // production, then throw a TypeError. // See, https://datatracker.ietf.org/doc/html/rfc7230#section-3.1.2: // reason-phrase = *( HTAB / SP / VCHAR / obs-text ) - // TODO + if (!isValidReasonPhrase(init.statusText)) { + throw new TypeError('Invalid statusText') + } } // 3. Set this’s response to a new response. diff --git a/lib/api/api-fetch/util.js b/lib/api/api-fetch/util.js index 1d8c5817f16..0014c582c8a 100644 --- a/lib/api/api-fetch/util.js +++ b/lib/api/api-fetch/util.js @@ -137,9 +137,30 @@ function requestBadPort (request) { return 'allowed' } +// Check whether |statusText| is a ByteString and +// matches the Reason-Phrase token production. +// RFC 2616: https://tools.ietf.org/html/rfc2616 +// RFC 7230: https://tools.ietf.org/html/rfc7230 +// "reason-phrase = *( HTAB / SP / VCHAR / obs-text )" +// https://github.com/chromium/chromium/blob/94.0.4604.1/third_party/blink/renderer/core/fetch/response.cc#L116 +function isValidReasonPhrase (statusText) { + for (let i = 0; i < statusText.length; ++i) { + const c = statusText.charCodeAt(i) + if (!( + c === 0x09 || // HTAB + (c >= 0x20 && c <= 0x7E) || // SP / VCHAR + (c >= 0x80 && c <= 0xFF) // obs-text + )) { + return false + } + } + return true +} + module.exports = { requestBadPort, requestCurrentURL, responseURL, - responseLocationURL + responseLocationURL, + isValidReasonPhrase } From e16955e3d224dd26fff242f81e9cb113826dae98 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 08:29:19 +0200 Subject: [PATCH 26/56] fixup: error messages --- lib/api/api-fetch/body.js | 8 ++------ lib/api/api-fetch/response.js | 8 ++++---- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/lib/api/api-fetch/body.js b/lib/api/api-fetch/body.js index 046bbe743fe..c2f37b881b6 100644 --- a/lib/api/api-fetch/body.js +++ b/lib/api/api-fetch/body.js @@ -101,12 +101,8 @@ function extractBody (object, keepalive = false) { } // If object is disturbed or locked, then throw a TypeError. - if (util.isDisturbed(stream)) { - throw new TypeError('disturbed') - } - - if (stream.locked) { - throw new TypeError('locked') + if (util.isDisturbed(stream) || stream.locked) { + throw new TypeError('Response body object should not be disturbed or locked') } } else if (typeof object === 'string') { // scalar value string diff --git a/lib/api/api-fetch/response.js b/lib/api/api-fetch/response.js index efad93c127b..5fb5edcb33d 100644 --- a/lib/api/api-fetch/response.js +++ b/lib/api/api-fetch/response.js @@ -39,14 +39,14 @@ class Response { try { parsedURL = new URL(url) } catch (err) { - const error = new TypeError() + const error = new TypeError('Failed to parse URL from ' + url) error.cause = err throw error } // 3. If status is not a redirect status, then throw a RangeError. if (!redirectStatus.includes(status)) { - throw new RangeError(`Failed to construct 'Response': The status provided (${status}) is outside the range [200, 599].`) + throw new RangeError('Invalid status code') } // 4. Let responseObject be the result of creating a Response object, @@ -79,7 +79,7 @@ class Response { } if (init.status < 200 || init.status > 599) { - throw new RangeError() + throw new RangeError(`Failed to construct 'Response': The status provided (${init.status}) is outside the range [200, 599].`) } } @@ -127,7 +127,7 @@ class Response { if (body !== null) { // 1. If init["status"] is a null body status, then throw a TypeError. if (nullBodyStatus.includes(init.status)) { - throw new TypeError() + throw new TypeError('Response with null body status cannot have body') } // 2. Let Content-Type be null. From c2eeb968a3a7a25c230bb5d79d99a66f23afe7fe Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 08:31:02 +0200 Subject: [PATCH 27/56] fixup: isBuffer --- lib/api/api-fetch/body.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/api/api-fetch/body.js b/lib/api/api-fetch/body.js index c2f37b881b6..27e8b8935e7 100644 --- a/lib/api/api-fetch/body.js +++ b/lib/api/api-fetch/body.js @@ -66,9 +66,6 @@ function extractBody (object, keepalive = false) { // Set source to a copy of the bytes held by object. source = new Uint8Array(object) - - // TODO: This is not part of spec. - length = source.byteLength } else if (object instanceof Blob) { // Blob @@ -120,7 +117,7 @@ function extractBody (object, keepalive = false) { // 7. If source is a byte sequence, then set action to a // step that returns source and length to source’s length. // TODO: What is a "byte sequence?" - if (typeof source === 'string') { + if (typeof source === 'string' || util.isBuffer(source)) { length = Buffer.byteLength(source) } From 301dc8ec42255cb4a7bc0b8e7bf19142adadec38 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 08:43:27 +0200 Subject: [PATCH 28/56] fixup: more error msg --- lib/api/api-fetch/request.js | 17 +++++++++-------- lib/api/api-fetch/util.js | 25 +++++++++++++++++++++++++ test/node-fetch/main.js | 4 ++-- 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/lib/api/api-fetch/request.js b/lib/api/api-fetch/request.js index 72d84928840..0221839cabe 100644 --- a/lib/api/api-fetch/request.js +++ b/lib/api/api-fetch/request.js @@ -6,6 +6,7 @@ const { METHODS } = require('http') const { extractBody, mixinBody, cloneBody } = require('./body') const { Headers, fill: fillHeaders, HeadersList } = require('./headers') const util = require('../../core/util') +const { isValidHTTPToken } = require('./util') const { corsSafeListedMethods, referrerPolicy, @@ -55,14 +56,14 @@ class Request { try { parsedURL = new URL(input, baseUrl) } catch (err) { - const error = new TypeError('Invalid URL') + const error = new TypeError('Failed to parse URL from ' + input) error.cause = err throw error } // 3. If parsedURL includes credentials, then throw a TypeError. if (parsedURL.username || parsedURL.password) { - throw new TypeError('Invalid URL') + throw new TypeError('Request cannot be constructed from a URL that includes credentials: ' + input) } // 4. Set request to a new request whose URL is parsedURL. @@ -146,7 +147,7 @@ class Request { try { parsedReferrer = new URL(referrer, baseUrl) } catch (err) { - const error = new TypeError('invalid referrer') + const error = new TypeError(`Referrer "${referrer}" is not a valid URL.`) error.cause = err throw error } @@ -177,7 +178,7 @@ class Request { if ('mode' in init) { mode = init.mode if (!requestMode.includes(mode)) { - throw new TypeError(`'mode' option '${mode}' is not a valid value of RequestMode`) + throw new TypeError('Cannot construct a Request with a RequestInit whose mode member is set as \'navigate\'.') } } else { mode = fallbackMode @@ -213,7 +214,7 @@ class Request { // 21. If request’s cache mode is "only-if-cached" and request’s mode is // not "same-origin", then throw a TypeError. if (request.cache === 'only-if-cached' && request.mode !== 'same-origin') { - throw new TypeError() + throw new TypeError('\'only-if-cached\' can be set only with \'same-origin\' mode') } // 22. If init["redirect"] exists, then set request’s redirect mode to it. @@ -247,12 +248,12 @@ class Request { // 2. If method is not a method or method is a forbidden method, then // throw a TypeError. - if (typeof init.method !== 'string') { - throw TypeError(`Request method: ${init.method} must be type 'string'`) + if (!isValidHTTPToken(init.method)) { + throw TypeError(`'${init.method}' is not a valid HTTP method.`) } if (METHODS.indexOf(method.toUpperCase()) === -1) { - throw Error(`Normalized request init.method: ${method} must be one of ${METHODS.join(', ')}`) + throw Error(`'${init.method}' HTTP method is unsupported.`) } // 3. Normalize method. diff --git a/lib/api/api-fetch/util.js b/lib/api/api-fetch/util.js index 0014c582c8a..86b90f40caf 100644 --- a/lib/api/api-fetch/util.js +++ b/lib/api/api-fetch/util.js @@ -157,7 +157,32 @@ function isValidReasonPhrase (statusText) { return true } +function isTokenChar (c) { + return !( + c >= 0x7F || c <= 0x20 || c === '(' || c === ')' || c === '<' || + c === '>' || c === '@' || c === ',' || c === ';' || c === ':' || + c === '\\' || c === '"' || c === '/' || c === '[' || c === ']' || + c === '?' || c === '=' || c === '{' || c === '}' + ) +} + +// See RFC 7230, Section 3.2.6. +// https://github.com/chromium/chromium/blob/d7da0240cae77824d1eda25745c4022757499131/third_party/blink/renderer/platform/network/http_parsers.cc#L321 +function isValidHTTPToken (characters) { + if (!characters || typeof characters !== 'string') { + return false + } + for (let i = 0; i < characters.length; ++i) { + const c = characters.charCodeAt(i) + if (c > 0x7F || !isTokenChar(c)) { + return false + } + } + return true +} + module.exports = { + isValidHTTPToken, requestBadPort, requestCurrentURL, responseURL, diff --git a/test/node-fetch/main.js b/test/node-fetch/main.js index ed5e36e11d4..b630c3b4024 100644 --- a/test/node-fetch/main.js +++ b/test/node-fetch/main.js @@ -82,12 +82,12 @@ describe('node-fetch', () => { it('should reject with error if url is protocol relative', () => { const url = '//example.com/' - return expect(fetch(url)).to.eventually.be.rejectedWith(TypeError, /Invalid URL/) + return expect(fetch(url)).to.eventually.be.rejectedWith(TypeError) }) it('should reject with error if url is relative path', () => { const url = '/some/path' - return expect(fetch(url)).to.eventually.be.rejectedWith(TypeError, /Invalid URL/) + return expect(fetch(url)).to.eventually.be.rejectedWith(TypeError) }) it('should reject with error if protocol is unsupported', () => { From 9122f57926edb51ea7916e27ce1b4ab0e0d7f0ea Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 08:47:04 +0200 Subject: [PATCH 29/56] fixup: forbidden methods --- lib/api/api-fetch/constants.js | 8 ++++++++ lib/api/api-fetch/request.js | 5 +++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/api/api-fetch/constants.js b/lib/api/api-fetch/constants.js index 1e750d87342..bcc07694e6b 100644 --- a/lib/api/api-fetch/constants.js +++ b/lib/api/api-fetch/constants.js @@ -103,8 +103,16 @@ const requestBodyHeader = [ 'content-type' ] +// http://fetch.spec.whatwg.org/#forbidden-method +const forbiddenMethods = [ + 'CONNECT', + 'TRACE', + 'TRACK' +] + module.exports = { forbiddenResponseHeaderNames, + forbiddenMethods, requestBodyHeader, referrerPolicy, requestRedirect, diff --git a/lib/api/api-fetch/request.js b/lib/api/api-fetch/request.js index 0221839cabe..3c9044bf13a 100644 --- a/lib/api/api-fetch/request.js +++ b/lib/api/api-fetch/request.js @@ -2,12 +2,12 @@ 'use strict' -const { METHODS } = require('http') const { extractBody, mixinBody, cloneBody } = require('./body') const { Headers, fill: fillHeaders, HeadersList } = require('./headers') const util = require('../../core/util') const { isValidHTTPToken } = require('./util') const { + forbiddenMethods, corsSafeListedMethods, referrerPolicy, requestRedirect, @@ -252,11 +252,12 @@ class Request { throw TypeError(`'${init.method}' is not a valid HTTP method.`) } - if (METHODS.indexOf(method.toUpperCase()) === -1) { + if (forbiddenMethods.indexOf(method.toUpperCase()) !== -1) { throw Error(`'${init.method}' HTTP method is unsupported.`) } // 3. Normalize method. + // https://fetch.spec.whatwg.org/#concept-method-normalize method = init.method.toUpperCase() // 4. Set request’s method to method. From 74fbc2052f71cfca0d131f0d911e4b4d5bc7e94e Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 09:52:45 +0200 Subject: [PATCH 30/56] fixup: node stream body does not violate spec --- lib/api/api-fetch/body.js | 7 +- lib/api/api-fetch/request.js | 4 - lib/api/api-fetch/response.js | 4 - lib/api/api-fetch/util.js | 158 ++++++++++++++++----------------- test/node-fetch/main.js | 5 +- test/node-fetch/response.js | 7 +- test/node-fetch/utils/toWeb.js | 73 --------------- 7 files changed, 90 insertions(+), 168 deletions(-) delete mode 100644 test/node-fetch/utils/toWeb.js diff --git a/lib/api/api-fetch/body.js b/lib/api/api-fetch/body.js index 27e8b8935e7..7a4a92bad11 100644 --- a/lib/api/api-fetch/body.js +++ b/lib/api/api-fetch/body.js @@ -1,6 +1,7 @@ 'use strict' const util = require('../../core/util') +const { toWebReadable } = require('./util') const { kState } = require('./symbols') const { Blob } = require('buffer') const { NotSupportedError } = require('../../core/errors') @@ -89,7 +90,7 @@ function extractBody (object, keepalive = false) { if (object.type) { contentType = object.type } - } else if (object instanceof ReadableStream) { + } else if (object instanceof ReadableStream || util.isStream(object)) { // ReadableStream // If keepalive is true, then throw a TypeError. @@ -101,6 +102,10 @@ function extractBody (object, keepalive = false) { if (util.isDisturbed(stream) || stream.locked) { throw new TypeError('Response body object should not be disturbed or locked') } + + if (util.isStream(object)) { + stream = toWebReadable(object) + } } else if (typeof object === 'string') { // scalar value string // TODO: How to check for "scalar value string"? diff --git a/lib/api/api-fetch/request.js b/lib/api/api-fetch/request.js index 3c9044bf13a..993aa246c83 100644 --- a/lib/api/api-fetch/request.js +++ b/lib/api/api-fetch/request.js @@ -425,10 +425,6 @@ class Request { return this.constructor.name } - toString () { - return Object.prototype.toString.call(this) - } - // Returns request’s HTTP method, which is "GET" by default. get method () { // The method getter steps are to return this’s request’s method. diff --git a/lib/api/api-fetch/response.js b/lib/api/api-fetch/response.js index 5fb5edcb33d..d36b26e33fd 100644 --- a/lib/api/api-fetch/response.js +++ b/lib/api/api-fetch/response.js @@ -149,10 +149,6 @@ class Response { return this.constructor.name } - toString () { - return Object.prototype.toString.call(this) - } - // Returns response’s type, e.g., "cors". get type () { // The type getter steps are to return this’s response’s type. diff --git a/lib/api/api-fetch/util.js b/lib/api/api-fetch/util.js index 86b90f40caf..4b67f8ed786 100644 --- a/lib/api/api-fetch/util.js +++ b/lib/api/api-fetch/util.js @@ -1,87 +1,20 @@ 'use strict' const { redirectStatus } = require('./constants') +const { destroy, isDestroyed } = require('../../../lib/core/util') +const { finished } = require('stream') +const { AbortError } = require('../../../lib/core/errors') + +let ReadableStream +let CountQueuingStrategy const badPorts = [ - 1, - 7, - 9, - 11, - 13, - 15, - 17, - 19, - 20, - 21, - 22, - 23, - 25, - 37, - 42, - 43, - 53, - 69, - 77, - 79, - 87, - 95, - 101, - 102, - 103, - 104, - 109, - 110, - 111, - 113, - 115, - 117, - 119, - 123, - 135, - 137, - 139, - 143, - 161, - 179, - 389, - 427, - 465, - 512, - 513, - 514, - 515, - 526, - 530, - 531, - 532, - 540, - 548, - 554, - 556, - 563, - 587, - 601, - 636, - 989, - 990, - 993, - 995, - 1719, - 1720, - 1723, - 2049, - 3659, - 4045, - 5060, - 5061, - 6000, - 6566, - 6665, - 6666, - 6667, - 6668, - 6669, - 6697, + 1, 7, 9, 11, 13, 15, 17, 19, 20, 21, 22, 23, 25, 37, 42, 43, + 53, 69, 77, 79, 87, 95, 101, 102, 103, 104, 109, 110, 111, 113, + 115, 117, 119, 123, 135, 137, 139, 143, 161, 179, 389, 427, 465, + 512, 513, 514, 515, 526, 530, 531, 532, 540, 548, 554, 556, 563, + 587, 601, 636, 989, 990, 993, 995, 1719, 1720, 1723, 2049, 3659, + 4045, 5060, 5061, 6000, 6566, 6665, 6666, 6667, 6668, 6669, 6697, 10080 ] @@ -181,7 +114,74 @@ function isValidHTTPToken (characters) { return true } +function toWebReadable (streamReadable) { + if (!ReadableStream) { + ReadableStream = require('stream/web').ReadableStream + } + if (!CountQueuingStrategy) { + CountQueuingStrategy = require('stream/web').CountQueuingStrategy + } + + if (isDestroyed(streamReadable)) { + const readable = new ReadableStream() + readable.cancel() + return readable + } + + const objectMode = streamReadable.readableObjectMode + const highWaterMark = streamReadable.readableHighWaterMark + const strategy = objectMode + ? new CountQueuingStrategy({ highWaterMark }) + : { highWaterMark } + + let controller + + function onData (chunk) { + // Copy the Buffer to detach it from the pool. + if (Buffer.isBuffer(chunk) && !objectMode) { + chunk = new Uint8Array(chunk) + } + controller.enqueue(chunk) + if (controller.desiredSize <= 0) { + streamReadable.pause() + } + } + + streamReadable.pause() + + finished(streamReadable, (err) => { + if (err && err.code === 'ERR_STREAM_PREMATURE_CLOSE') { + const er = new AbortError() + er.cause = er + err = er + } + + if (err) { + controller.error(err) + } else { + controller.close() + } + }) + + streamReadable.on('data', onData) + + return new ReadableStream({ + start (c) { + controller = c + }, + + pull () { + streamReadable.resume() + }, + + cancel (reason) { + destroy(streamReadable, reason) + } + }, strategy) +} + module.exports = { + toWebReadable, isValidHTTPToken, requestBadPort, requestCurrentURL, diff --git a/test/node-fetch/main.js b/test/node-fetch/main.js index b630c3b4024..3895093dc1d 100644 --- a/test/node-fetch/main.js +++ b/test/node-fetch/main.js @@ -26,7 +26,6 @@ const RequestOrig = require('../../lib/api/api-fetch/request.js').Request const ResponseOrig = require('../../lib/api/api-fetch/response.js').Response const TestServer = require('./utils/server.js') const chaiTimeout = require('./utils/chai-timeout.js') -const toWeb = require('./utils/toWeb.js') const { ReadableStream } = require('stream/web') function isNodeLowerThan (version) { @@ -346,7 +345,7 @@ describe('node-fetch', () => { const url = `${base}redirect/307` const options = { method: 'PATCH', - body: toWeb(stream.Readable.from('tada')) + body: stream.Readable.from('tada') } return expect(fetch(url, options)).to.eventually.be.rejected .and.be.an.instanceOf(TypeError) @@ -1128,7 +1127,7 @@ describe('node-fetch', () => { const url = `${base}inspect` const options = { method: 'POST', - body: toWeb(stream.Readable.from('a=1')) + body: stream.Readable.from('a=1') } return fetch(url, options).then(res => { return res.json() diff --git a/test/node-fetch/response.js b/test/node-fetch/response.js index 42e477f1b73..71bc24e4052 100644 --- a/test/node-fetch/response.js +++ b/test/node-fetch/response.js @@ -4,7 +4,6 @@ const chai = require('chai') const stream = require('stream') const { Response } = require('../../lib/api/api-fetch/response.js') const TestServer = require('./utils/server.js') -const toWeb = require('./utils/toWeb.js') const { Blob } = require('buffer') const { kState } = require('../../lib/api/api-fetch/symbols.js') @@ -68,7 +67,7 @@ describe('Response', () => { }) it('should support empty options', () => { - const res = new Response(toWeb(stream.Readable.from('a=1'))) + const res = new Response(stream.Readable.from('a=1')) return res.text().then(result => { expect(result).to.equal('a=1') }) @@ -114,7 +113,7 @@ describe('Response', () => { } xit('should support clone() method', () => { - const body = toWeb(stream.Readable.from('a=1')) + const body = stream.Readable.from('a=1') const res = new Response(body, { headers: { a: '1' @@ -139,7 +138,7 @@ describe('Response', () => { }) it('should support stream as body', () => { - const body = toWeb(stream.Readable.from('a=1')) + const body = stream.Readable.from('a=1') const res = new Response(body) return res.text().then(result => { expect(result).to.equal('a=1') diff --git a/test/node-fetch/utils/toWeb.js b/test/node-fetch/utils/toWeb.js deleted file mode 100644 index 3639146019d..00000000000 --- a/test/node-fetch/utils/toWeb.js +++ /dev/null @@ -1,73 +0,0 @@ -let ReadableStream -let CountQueuingStrategy - -const { destroy, isDestroyed } = require('../../../lib/core/util') - -const { finished } = require('stream') -const { AbortError } = require('../../../lib/core/errors') - -module.exports = function toWeb (streamReadable) { - if (!ReadableStream) { - ReadableStream = require('stream/web').ReadableStream - } - if (!CountQueuingStrategy) { - CountQueuingStrategy = require('stream/web').CountQueuingStrategy - } - - if (isDestroyed(streamReadable)) { - const readable = new ReadableStream() - readable.cancel() - return readable - } - - const objectMode = streamReadable.readableObjectMode - const highWaterMark = streamReadable.readableHighWaterMark - const strategy = objectMode - ? new CountQueuingStrategy({ highWaterMark }) - : { highWaterMark } - - let controller - - function onData (chunk) { - // Copy the Buffer to detach it from the pool. - if (Buffer.isBuffer(chunk) && !objectMode) { - chunk = new Uint8Array(chunk) - } - controller.enqueue(chunk) - if (controller.desiredSize <= 0) { - streamReadable.pause() - } - } - - streamReadable.pause() - - finished(streamReadable, (err) => { - if (err && err.code === 'ERR_STREAM_PREMATURE_CLOSE') { - const er = new AbortError() - er.cause = er - err = er - } - - if (err) { - controller.error(err) - } else { - controller.close() - } - }) - - streamReadable.on('data', onData) - - return new ReadableStream({ - start (c) { - controller = c - }, - - pull () { - streamReadable.resume() - }, - - cancel (reason) { - destroy(streamReadable, reason) - } - }, strategy) -} From 25485a0e2da935127e7f483cd25b43e6b3a3e09a Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 09:57:45 +0200 Subject: [PATCH 31/56] fixup: error msgs --- lib/api/api-fetch/request.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/api/api-fetch/request.js b/lib/api/api-fetch/request.js index 993aa246c83..0e4d9cd4199 100644 --- a/lib/api/api-fetch/request.js +++ b/lib/api/api-fetch/request.js @@ -310,7 +310,7 @@ class Request { // 1. If this’s request’s method is not a CORS-safelisted method, // then throw a TypeError. if (!corsSafeListedMethods.includes(request.method)) { - throw new TypeError() + throw new TypeError(`'${request.method} is unsupported in no-cors mode.`) } // 2. Set this’s headers’s guard to "request-no-cors". @@ -353,7 +353,7 @@ class Request { (('body' in init && init !== null) || inputBody !== null) && (request.method === 'GET' || request.method === 'HEAD') ) { - throw new TypeError() + throw new TypeError('Request with GET/HEAD method cannot have body.') } // 35. Let initBody be null. From 3344db1b15fb0f424b1feca1aa1824d18da60e69 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 10:04:21 +0200 Subject: [PATCH 32/56] fixup --- lib/api/api-fetch/body.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/api/api-fetch/body.js b/lib/api/api-fetch/body.js index 7a4a92bad11..2a3cd27100e 100644 --- a/lib/api/api-fetch/body.js +++ b/lib/api/api-fetch/body.js @@ -4,6 +4,7 @@ const util = require('../../core/util') const { toWebReadable } = require('./util') const { kState } = require('./symbols') const { Blob } = require('buffer') +const { Readable } = require('stream') const { NotSupportedError } = require('../../core/errors') const { ReadableStream } = require('stream/web') const { kBodyUsed } = require('../../core/symbols') @@ -90,7 +91,7 @@ function extractBody (object, keepalive = false) { if (object.type) { contentType = object.type } - } else if (object instanceof ReadableStream || util.isStream(object)) { + } else if (object instanceof ReadableStream || object instanceof Readable) { // ReadableStream // If keepalive is true, then throw a TypeError. From 9f2fd140fb5a69a967515fd24b6f979736050b51 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 10:06:00 +0200 Subject: [PATCH 33/56] fixup --- lib/api/api-fetch/body.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/api/api-fetch/body.js b/lib/api/api-fetch/body.js index 2a3cd27100e..c25dec297e4 100644 --- a/lib/api/api-fetch/body.js +++ b/lib/api/api-fetch/body.js @@ -3,13 +3,13 @@ const util = require('../../core/util') const { toWebReadable } = require('./util') const { kState } = require('./symbols') -const { Blob } = require('buffer') -const { Readable } = require('stream') +const { Blob } = require('node:buffer') +const { Readable } = require('node:stream') const { NotSupportedError } = require('../../core/errors') -const { ReadableStream } = require('stream/web') +const { ReadableStream } = require('node:stream/web') const { kBodyUsed } = require('../../core/symbols') const assert = require('assert') -const nodeUtil = require('util') +const nodeUtil = require('node:util') // https://fetch.spec.whatwg.org/#concept-bodyinit-extract function extractBody (object, keepalive = false) { From 3bcf1a3f98b9975b71ccc00e60af227335538699 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 10:07:30 +0200 Subject: [PATCH 34/56] fixup --- lib/api/api-fetch/request.js | 2 +- lib/api/api-fetch/response.js | 2 +- lib/api/api-fetch/util.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/api/api-fetch/request.js b/lib/api/api-fetch/request.js index 0e4d9cd4199..9d5b20a53bb 100644 --- a/lib/api/api-fetch/request.js +++ b/lib/api/api-fetch/request.js @@ -23,7 +23,7 @@ const { kGuard } = require('./symbols') const { kHeadersList } = require('../../core/symbols') -const assert = require('assert') +const assert = require('node:assert') let TransformStream diff --git a/lib/api/api-fetch/response.js b/lib/api/api-fetch/response.js index d36b26e33fd..3268ca420af 100644 --- a/lib/api/api-fetch/response.js +++ b/lib/api/api-fetch/response.js @@ -6,13 +6,13 @@ const util = require('../../core/util') const { kEnumerableProperty } = util const { responseURL, isValidReasonPhrase } = require('./util') const { redirectStatus, nullBodyStatus, forbiddenHeaderNames } = require('./constants') -const assert = require('assert') const { kState, kHeaders, kGuard } = require('./symbols') const { kHeadersList } = require('../../core/symbols') +const assert = require('node:assert') // https://fetch.spec.whatwg.org/#response-class class Response { diff --git a/lib/api/api-fetch/util.js b/lib/api/api-fetch/util.js index 4b67f8ed786..74a77103972 100644 --- a/lib/api/api-fetch/util.js +++ b/lib/api/api-fetch/util.js @@ -2,8 +2,8 @@ const { redirectStatus } = require('./constants') const { destroy, isDestroyed } = require('../../../lib/core/util') -const { finished } = require('stream') const { AbortError } = require('../../../lib/core/errors') +const { finished } = require('node:stream') let ReadableStream let CountQueuingStrategy From b397500c7b8a782377c9020fe92f3d85d476365b Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 10:20:19 +0200 Subject: [PATCH 35/56] fixup --- lib/api/api-fetch/response.js | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/api/api-fetch/response.js b/lib/api/api-fetch/response.js index 3268ca420af..fba70c1407f 100644 --- a/lib/api/api-fetch/response.js +++ b/lib/api/api-fetch/response.js @@ -84,15 +84,11 @@ class Response { } if ('statusText' in init) { - if (typeof init.statusText !== 'string') { - throw new TypeError('Invalid statusText') - } - // 2. If init["statusText"] does not match the reason-phrase token // production, then throw a TypeError. // See, https://datatracker.ietf.org/doc/html/rfc7230#section-3.1.2: // reason-phrase = *( HTAB / SP / VCHAR / obs-text ) - if (!isValidReasonPhrase(init.statusText)) { + if (!isValidReasonPhrase(String(init.statusText))) { throw new TypeError('Invalid statusText') } } @@ -115,7 +111,7 @@ class Response { // 6. Set this’s response’s status message to init["statusText"]. if ('statusText' in init) { - this[kState].statusText = init.statusText + this[kState].statusText = String(init.statusText) } // 7. If init["headers"] exists, then fill this’s headers with init["headers"]. From 52f6f8335cf26a6f966dd95ec4396f18e4997f29 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 10:25:26 +0200 Subject: [PATCH 36/56] fixup: chrome compat --- lib/api/api-fetch/request.js | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/lib/api/api-fetch/request.js b/lib/api/api-fetch/request.js index 9d5b20a53bb..782018fe142 100644 --- a/lib/api/api-fetch/request.js +++ b/lib/api/api-fetch/request.js @@ -102,10 +102,6 @@ class Request { // 11. If init["window"] exists, then set window to "no-window". if ('window' in init) { window = 'no-window' - - if (window !== null) { - throw new TypeError(`'window' option '${window}' must be null`) - } } // 12. Set request to a new request with the following properties: @@ -169,7 +165,7 @@ class Request { if ('referrerPolicy' in init) { request.referrerPolicy = init.referrerPolicy if (!referrerPolicy.includes(request.referrerPolicy)) { - throw new TypeError(`'referrer' option '${request.referrerPolicy}' is not a valid value of ReferrerPolicy`) + throw new TypeError(`Failed to construct 'Request': The provided value '${request.referrerPolicy}' is not a valid enum value of type ReferrerPolicy.`) } } @@ -178,7 +174,7 @@ class Request { if ('mode' in init) { mode = init.mode if (!requestMode.includes(mode)) { - throw new TypeError('Cannot construct a Request with a RequestInit whose mode member is set as \'navigate\'.') + throw new TypeError(`Failed to construct 'Request': The provided value '${request.mode}' is not a valid enum value of type RequestMode.`) } } else { mode = fallbackMode @@ -199,7 +195,7 @@ class Request { if ('credentials' in init) { request.credentials = init.credentials if (!requestCredentials.includes(request.credentials)) { - throw new TypeError(`'credentials' option '${request.credentials}' is not a valid value of RequestCredentials`) + throw new TypeError(`Failed to construct 'Request': The provided value '${request.credentials}' is not a valid enum value of type RequestCredentials.`) } } @@ -207,7 +203,7 @@ class Request { if ('cache' in init) { request.cache = init.cache if (!requestCache.includes(request.cache)) { - throw new TypeError(`'cache' option '${request.cache}' is not a valid value of RequestCache`) + throw new TypeError(`Failed to construct 'Request': The provided value '${request.cache}' is not a valid enum value of type RequestCache.`) } } @@ -221,24 +217,18 @@ class Request { if ('redirect' in init) { request.redirect = init.redirect if (!requestRedirect.includes(request.redirect)) { - throw new TypeError(`'redirect' option '${request.redirect}' is not a valid value of RequestRedirect`) + throw new TypeError(`Failed to construct 'Request': The provided value '${request.redirect}' is not a valid enum value of type RequestRedirect.`) } } // 23. If init["integrity"] exists, then set request’s integrity metadata to it. if ('integrity' in init) { - request.integrity = init.integrity - if (typeof request.integrity !== 'string') { - throw new TypeError(`'integrity' option '${request.integrity}' is not a valid value of string`) - } + request.integrity = String(init.integrity) } // 24. If init["keepalive"] exists, then set request’s keepalive to it. if ('keepalive' in init) { - request.keepalive = init.keepalive - if (typeof request.keepalive !== 'boolean') { - throw new TypeError(`'keepalive' option '${request.keepalive}' is not a valid value of boolean`) - } + request.keepalive = Boolean(init.keepalive) } // 25. If init["method"] exists, then: @@ -284,7 +274,7 @@ class Request { !signal || typeof signal.aborted !== 'boolean' || typeof signal.addEventListener !== 'function') { - throw new TypeError(`'signal' option '${signal}' is not a valid value of AbortSignal`) + throw new TypeError('Failed to construct \'Request\': member signal is not of type AbortSignal.') } if (signal.aborted) { From a8a9b6aaac790fd05ce5a3da4e6d3283323d81ef Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 10:27:49 +0200 Subject: [PATCH 37/56] fixup --- lib/api/api-fetch/body.js | 8 ++++---- lib/api/api-fetch/request.js | 4 ++-- lib/api/api-fetch/response.js | 2 +- lib/api/api-fetch/util.js | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/api/api-fetch/body.js b/lib/api/api-fetch/body.js index c25dec297e4..2a3cd27100e 100644 --- a/lib/api/api-fetch/body.js +++ b/lib/api/api-fetch/body.js @@ -3,13 +3,13 @@ const util = require('../../core/util') const { toWebReadable } = require('./util') const { kState } = require('./symbols') -const { Blob } = require('node:buffer') -const { Readable } = require('node:stream') +const { Blob } = require('buffer') +const { Readable } = require('stream') const { NotSupportedError } = require('../../core/errors') -const { ReadableStream } = require('node:stream/web') +const { ReadableStream } = require('stream/web') const { kBodyUsed } = require('../../core/symbols') const assert = require('assert') -const nodeUtil = require('node:util') +const nodeUtil = require('util') // https://fetch.spec.whatwg.org/#concept-bodyinit-extract function extractBody (object, keepalive = false) { diff --git a/lib/api/api-fetch/request.js b/lib/api/api-fetch/request.js index 782018fe142..535035aad81 100644 --- a/lib/api/api-fetch/request.js +++ b/lib/api/api-fetch/request.js @@ -23,7 +23,7 @@ const { kGuard } = require('./symbols') const { kHeadersList } = require('../../core/symbols') -const assert = require('node:assert') +const assert = require('assert') let TransformStream @@ -323,7 +323,7 @@ class Request { this[kHeaders][kHeadersList] = this[kState].headersList // 4. If headers is a Headers object, then for each header in its header - // list, append header’s name/header’s value to this’s headers. + // list, append header’s name/header’s value to this’s headers. if (headers instanceof Headers) { this[kState].headersList.push(...headers[kHeadersList]) } else { diff --git a/lib/api/api-fetch/response.js b/lib/api/api-fetch/response.js index fba70c1407f..6d3fad9dbb7 100644 --- a/lib/api/api-fetch/response.js +++ b/lib/api/api-fetch/response.js @@ -12,7 +12,7 @@ const { kGuard } = require('./symbols') const { kHeadersList } = require('../../core/symbols') -const assert = require('node:assert') +const assert = require('assert') // https://fetch.spec.whatwg.org/#response-class class Response { diff --git a/lib/api/api-fetch/util.js b/lib/api/api-fetch/util.js index 74a77103972..849972c734c 100644 --- a/lib/api/api-fetch/util.js +++ b/lib/api/api-fetch/util.js @@ -3,7 +3,7 @@ const { redirectStatus } = require('./constants') const { destroy, isDestroyed } = require('../../../lib/core/util') const { AbortError } = require('../../../lib/core/errors') -const { finished } = require('node:stream') +const { finished } = require('stream') let ReadableStream let CountQueuingStrategy From 4ce55d394d134fb712ac6c3a240cf768143458a5 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 10:30:07 +0200 Subject: [PATCH 38/56] fixup: error msgs --- lib/api/api-fetch/request.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/api/api-fetch/request.js b/lib/api/api-fetch/request.js index 535035aad81..d19ae663361 100644 --- a/lib/api/api-fetch/request.js +++ b/lib/api/api-fetch/request.js @@ -375,7 +375,7 @@ class Request { // 1. If this’s request’s mode is neither "same-origin" nor "cors", // then throw a TypeError. if (request.mode !== 'same-origin' && request.mode !== 'cors') { - throw new TypeError() + throw new TypeError('If request is made from ReadableStream, mode should be "same-origin" or "cors"') } // 2. Set this’s request’s use-CORS-preflight flag. @@ -389,7 +389,7 @@ class Request { if (initBody == null && inputBody !== null) { // 1. If input is unusable, then throw a TypeError. if (util.isDisturbed(inputBody.stream) || inputBody.stream.locked) { - throw new TypeError('unusable') + throw new TypeError('Cannot construct a Request with a Request object that has already been used.') } // 2. Set finalBody to the result of creating a proxy for inputBody. From af5866a3391d5d79c979db9201abb7aed73620fb Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 10:33:52 +0200 Subject: [PATCH 39/56] fixup --- lib/api/api-fetch/index.js | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/api/api-fetch/index.js b/lib/api/api-fetch/index.js index 5a8aba53ccc..8c92542e4b6 100644 --- a/lib/api/api-fetch/index.js +++ b/lib/api/api-fetch/index.js @@ -17,8 +17,7 @@ const { requestBodyHeader } = require('./constants') const { kHeadersList } = require('../../core/symbols') - -let ReadableStream +const { ReadableStream } = require('stream/web') // https://fetch.spec.whatwg.org/#fetch-method async function fetch (resource, init) { @@ -1191,10 +1190,6 @@ function httpNetworkFetch ( headers.append(headersList[n + 0].toString(), headersList[n + 1].toString()) } - if (!ReadableStream) { - ReadableStream = require('stream/web').ReadableStream - } - const stream = status === 204 ? null : new ReadableStream({ From 9f6d2e03dcb60e3f63d516b3805ad0ae0a7ad766 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 10:41:03 +0200 Subject: [PATCH 40/56] fixup: timingInfo --- lib/api/api-fetch/index.js | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/lib/api/api-fetch/index.js b/lib/api/api-fetch/index.js index 8c92542e4b6..e00e89ebde2 100644 --- a/lib/api/api-fetch/index.js +++ b/lib/api/api-fetch/index.js @@ -18,6 +18,7 @@ const { } = require('./constants') const { kHeadersList } = require('../../core/symbols') const { ReadableStream } = require('stream/web') +const { performance } = require('perf_hooks') // https://fetch.spec.whatwg.org/#fetch-method async function fetch (resource, init) { @@ -197,7 +198,21 @@ function fetching ({ // 5. Let timingInfo be a new fetch timing info whose start time and // post-redirect start time are the coarsened shared current time given // crossOriginIsolatedCapability. - // TODO + // TODO: Coarsened shared current time given crossOriginIsolatedCapability? + const currenTime = performance.now() + const timingInfo = { + startTime: currenTime, + redirectStartTime: 0, + redirectEndTime: 0, + postRedirectStartTime: currenTime, + finalServiceWorkerStartTime: 0, + finalNetworkResponseStartTime: 0, + finalNetworkRequestStartTime: 0, + endTime: 0, + encodedBodySize: 0, + decodedBodySize: 0, + finalConnectionTimingInfo: null + } // 6. Let fetchParams be a new fetch params whose request is request, timing // info is timingInfo, process request body is processRequestBody, @@ -208,7 +223,7 @@ function fetching ({ // is crossOriginIsolatedCapability. const fetchParams = { request, - timingInfo: null, + timingInfo, processRequestBody: null, processRequestEndOfBody: null, processResponse, @@ -760,15 +775,18 @@ async function httpRedirectFetch (fetchParams, response) { } // 15. Let timingInfo be fetchParams’s timing info. - // TODO + const timingInfo = fetchParams.timingInfo // 16. Set timingInfo’s redirect end time and post-redirect start time to the // coarsened shared current time given fetchParams’s cross-origin isolated capability. - // TODO + // TODO: given fetchParams’s cross-origin isolated capability? + timingInfo.redirectEndTime = timingInfo.postRedirectStartTime = performance.now() // 17. If timingInfo’s redirect start time is 0, then set timingInfo’s redirect start // time to timingInfo’s start time. - // TODO + if (timingInfo.redirectStartTime === 0) { + timingInfo.redirectStartTime = timingInfo.startTime + } // 18. Append locationURL to request’s URL list. request.urlList.push(locationURL) From 1e87e8df43d8747e69bae2903f0a15d511da39c2 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 10:43:06 +0200 Subject: [PATCH 41/56] fixup: comments --- lib/api/api-fetch/index.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/api/api-fetch/index.js b/lib/api/api-fetch/index.js index e00e89ebde2..70df2f7b809 100644 --- a/lib/api/api-fetch/index.js +++ b/lib/api/api-fetch/index.js @@ -356,8 +356,7 @@ async function mainFetch (fetchParams, recursive = false) { // - request’s mode is "navigate" or "websocket" // 1. Set request’s response tainting to "basic". // 2. Return the result of running scheme fetch given fetchParams. - // NOTE: This branch is not possible since we cannot set request mode - // to "navigate". + // TODO // request’s mode is "same-origin" if (request.mode === 'same-origin') { @@ -427,10 +426,12 @@ async function mainFetch (fetchParams, recursive = false) { // 1. Let headerNames be the result of extracting header list values // given `Access-Control-Expose-Headers` and response’s header list. // TODO + // 2. If request’s credentials mode is not "include" and headerNames // contains `*`, then set response’s CORS-exposed header-name list to // all unique header names in response’s header list. // TODO + // 3. Otherwise, if headerNames is not null or failure, then set // response’s CORS-exposed header-name list to headerNames. // TODO From 7bd95d51ee073b4370e5668f898e878b687c7713 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 10:47:19 +0200 Subject: [PATCH 42/56] fixup: prettier + standard --fix --- lib/api/api-fetch/body.js | 40 ++-- lib/api/api-fetch/constants.js | 58 +----- lib/api/api-fetch/headers.js | 18 +- lib/api/api-fetch/index.js | 330 ++++++++++++++++++--------------- lib/api/api-fetch/request.js | 88 ++++++--- lib/api/api-fetch/response.js | 30 +-- lib/api/api-fetch/util.js | 75 +++++--- 7 files changed, 351 insertions(+), 288 deletions(-) diff --git a/lib/api/api-fetch/body.js b/lib/api/api-fetch/body.js index 2a3cd27100e..0b4bc1b1d24 100644 --- a/lib/api/api-fetch/body.js +++ b/lib/api/api-fetch/body.js @@ -22,10 +22,8 @@ function extractBody (object, keepalive = false) { async start (c) { controller = c }, - async pull () { - }, - async cancel (reason) { - } + async pull () {}, + async cancel (reason) {} }) } @@ -101,7 +99,9 @@ function extractBody (object, keepalive = false) { // If object is disturbed or locked, then throw a TypeError. if (util.isDisturbed(stream) || stream.locked) { - throw new TypeError('Response body object should not be disturbed or locked') + throw new TypeError( + 'Response body object should not be disturbed or locked' + ) } if (util.isStream(object)) { @@ -130,20 +130,24 @@ function extractBody (object, keepalive = false) { // 8. If action is non-null, then run these steps in in parallel: if (action !== null) { // Run action. - action(bytes => { - // Whenever one or more bytes are available and stream is not errored, - // enqueue a Uint8Array wrapping an ArrayBuffer containing the available - // bytes into stream. - if (!/state: 'errored'/.test(nodeUtil.inspect(stream))) { - controller.enqueue(new Uint8Array(bytes)) + action( + (bytes) => { + // Whenever one or more bytes are available and stream is not errored, + // enqueue a Uint8Array wrapping an ArrayBuffer containing the available + // bytes into stream. + if (!/state: 'errored'/.test(nodeUtil.inspect(stream))) { + controller.enqueue(new Uint8Array(bytes)) + } + }, + (err) => { + // TODO: Spec doesn't say anything about this? + controller.error(err) + }, + () => { + // When running action is done, close stream. + controller.close() } - }, err => { - // TODO: Spec doesn't say anything about this? - controller.error(err) - }, () => { - // When running action is done, close stream. - controller.close() - }) + ) } else if (controller) { // TODO: Spec doesn't say anything about this? controller.enqueue(source) diff --git a/lib/api/api-fetch/constants.js b/lib/api/api-fetch/constants.js index bcc07694e6b..f40145ab978 100644 --- a/lib/api/api-fetch/constants.js +++ b/lib/api/api-fetch/constants.js @@ -23,26 +23,11 @@ const forbiddenHeaderNames = [ 'via' ] -const corsSafeListedMethods = [ - 'GET', - 'HEAD', - 'POST' -] +const corsSafeListedMethods = ['GET', 'HEAD', 'POST'] -const nullBodyStatus = [ - 101, - 204, - 205, - 304 -] +const nullBodyStatus = [101, 204, 205, 304] -const redirectStatus = [ - 301, - 302, - 303, - 307, - 308 -] +const redirectStatus = [301, 302, 303, 307, 308] const referrerPolicy = [ '', @@ -56,31 +41,13 @@ const referrerPolicy = [ 'unsafe-url' ] -const requestRedirect = [ - 'follow', - 'manual', - 'error' -] +const requestRedirect = ['follow', 'manual', 'error'] -const safeMethods = [ - 'GET', - 'HEAD', - 'OPTIONS', - 'TRACE' -] +const safeMethods = ['GET', 'HEAD', 'OPTIONS', 'TRACE'] -const requestMode = [ - 'navigate', - 'same-origin', - 'no-cors', - 'cors' -] +const requestMode = ['navigate', 'same-origin', 'no-cors', 'cors'] -const requestCredentials = [ - 'omit', - 'same-origin', - 'include' -] +const requestCredentials = ['omit', 'same-origin', 'include'] const requestCache = [ 'default', @@ -91,10 +58,7 @@ const requestCache = [ 'only-if-cached' ] -const forbiddenResponseHeaderNames = [ - 'set-cookie', - 'set-cookie2' -] +const forbiddenResponseHeaderNames = ['set-cookie', 'set-cookie2'] const requestBodyHeader = [ 'content-encoding', @@ -104,11 +68,7 @@ const requestBodyHeader = [ ] // http://fetch.spec.whatwg.org/#forbidden-method -const forbiddenMethods = [ - 'CONNECT', - 'TRACE', - 'TRACK' -] +const forbiddenMethods = ['CONNECT', 'TRACE', 'TRACK'] module.exports = { forbiddenResponseHeaderNames, diff --git a/lib/api/api-fetch/headers.js b/lib/api/api-fetch/headers.js index e9af0743c41..abd50e76e45 100644 --- a/lib/api/api-fetch/headers.js +++ b/lib/api/api-fetch/headers.js @@ -46,7 +46,10 @@ function normalizeAndValidateHeaderValue (name, value) { if (value === undefined) { throw new HTTPInvalidHeaderValueError(value, name) } - const normalizedHeaderValue = `${value}`.replace(/^[\n\t\r\x20]+|[\n\t\r\x20]+$/g, '') + const normalizedHeaderValue = `${value}`.replace( + /^[\n\t\r\x20]+|[\n\t\r\x20]+$/g, + '' + ) validateHeaderValue(name, normalizedHeaderValue) return normalizedHeaderValue } @@ -99,7 +102,11 @@ function fill (headers, object) { function validateArgumentLength (found, expected) { if (found !== expected) { - throw new TypeError(`${expected} ${expected > 1 ? 'arguments' : 'argument'} required, but only ${found} present`) + throw new TypeError( + `${expected} ${ + expected > 1 ? 'arguments' : 'argument' + } required, but only ${found} present` + ) } } @@ -330,7 +337,12 @@ class Headers { } for (let index = 0; index < this[kHeadersList].length; index += 2) { - callback.call(thisArg, this[kHeadersList][index + 1], this[kHeadersList][index], this) + callback.call( + thisArg, + this[kHeadersList][index + 1], + this[kHeadersList][index], + this + ) } } diff --git a/lib/api/api-fetch/index.js b/lib/api/api-fetch/index.js index 70df2f7b809..7071369c683 100644 --- a/lib/api/api-fetch/index.js +++ b/lib/api/api-fetch/index.js @@ -2,10 +2,19 @@ 'use strict' -const { Response, makeNetworkError, filterResponse, makeResponse } = require('./response') +const { + Response, + makeNetworkError, + filterResponse, + makeResponse +} = require('./response') const { Headers } = require('./headers') const { Request, makeRequest } = require('./request') -const { requestBadPort, responseLocationURL, requestCurrentURL } = require('./util') +const { + requestBadPort, + responseLocationURL, + requestCurrentURL +} = require('./util') const { kState, kHeaders, kGuard } = require('./symbols') const { AbortError } = require('../../core/errors') const assert = require('assert') @@ -80,20 +89,25 @@ async function fetch (resource, init) { let locallyAborted = false // 10. Add the following abort steps to requestObject’s signal: - requestObject.signal.addEventListener('abort', () => { - // 1. Set locallyAborted to true. - locallyAborted = true + requestObject.signal.addEventListener( + 'abort', + () => { + // 1. Set locallyAborted to true. + locallyAborted = true - // 2. Abort fetch with p, request, and responseObject. - abortFetch.call(context, p, request, responseObject) + // 2. Abort fetch with p, request, and responseObject. + abortFetch.call(context, p, request, responseObject) - // 3. Terminate the ongoing fetch with the aborted flag set. - context.terminate({ aborted: true }) - }, { once: true }) + // 3. Terminate the ongoing fetch with the aborted flag set. + context.terminate({ aborted: true }) + }, + { once: true } + ) // 11. Let handleFetchDone given response response be to finalize and // report timing with response, globalObject, and "fetch". - const handleFetchDone = (response) => finalizeAndReportTiming(response, 'fetch') + const handleFetchDone = (response) => + finalizeAndReportTiming(response, 'fetch') // 12. Fetch request with processResponseDone set to handleFetchDone, // and processResponse given response being these substeps: @@ -177,11 +191,7 @@ function abortFetch (p, request, responseObject) { } // https://fetch.spec.whatwg.org/#fetching -function fetching ({ - request, - processResponse, - processResponseDone -}) { +function fetching ({ request, processResponse, processResponseDone }) { // 1. Let taskDestination be null. // TODO @@ -302,7 +312,10 @@ async function mainFetch (fetchParams, recursive = false) { // 3. If request’s local-URLs-only flag is set and request’s current URL is // not local, then set response to a network error. - if (request.localURLsOnly && !/^(about|blob|data):/.test(requestCurrentURL(request).protocol)) { + if ( + request.localURLsOnly && + !/^(about|blob|data):/.test(requestCurrentURL(request).protocol) + ) { return makeNetworkError('local URLs only') } @@ -369,7 +382,9 @@ async function mainFetch (fetchParams, recursive = false) { // 1. If request’s redirect mode is not "follow", then return a network // error. if (request.redirect !== 'follow') { - return makeNetworkError('redirect cmode cannot be "follow" for "no-cors" request') + return makeNetworkError( + 'redirect cmode cannot be "follow" for "no-cors" request' + ) } // 2. Set request’s response tainting to "opaque". @@ -410,7 +425,9 @@ async function mainFetch (fetchParams, recursive = false) { request.responseTainting = 'cors' // 2. Return the result of running HTTP fetch given fetchParams. - return await httpFetch.call(this, fetchParams).catch((err) => makeNetworkError(err)) + return await httpFetch + .call(this, fetchParams) + .catch((err) => makeNetworkError(err)) })() // 12. If recursive is true, then return response. @@ -426,12 +443,10 @@ async function mainFetch (fetchParams, recursive = false) { // 1. Let headerNames be the result of extracting header list values // given `Access-Control-Expose-Headers` and response’s header list. // TODO - // 2. If request’s credentials mode is not "include" and headerNames // contains `*`, then set response’s CORS-exposed header-name list to // all unique header names in response’s header list. // TODO - // 3. Otherwise, if headerNames is not null or failure, then set // response’s CORS-exposed header-name list to headerNames. // TODO @@ -452,9 +467,8 @@ async function mainFetch (fetchParams, recursive = false) { // 14. Let internalResponse be response, if response is a network error, // and response’s internal response otherwise. - let internalResponse = response.status === 0 - ? response - : response.internalResponse + let internalResponse = + response.status === 0 ? response : response.internalResponse // 15. If internalResponse’s URL list is empty, then set it to a clone of // request’s URL list. @@ -495,11 +509,9 @@ async function mainFetch (fetchParams, recursive = false) { // it (if any). if ( response.status !== 0 && - ( - request.method === 'HEAD' || + (request.method === 'HEAD' || request.method === 'CONNECT' || - nullBodyStatus.includes(internalResponse.status) - ) + nullBodyStatus.includes(internalResponse.status)) ) { internalResponse.body = null if (context.abort) { @@ -511,15 +523,13 @@ async function mainFetch (fetchParams, recursive = false) { if (request.integrity) { // 1. Let processBodyError be this step: run fetch finale given fetchParams // and a network error. - const processBodyError = (reason) => fetchFinale(fetchParams, makeNetworkError(reason)) + const processBodyError = (reason) => + fetchFinale(fetchParams, makeNetworkError(reason)) // 2. If request’s response tainting is "opaque", response is a network // error, or response’s body is null, then run processBodyError and abort // these steps. - if ( - request.responseTainting === 'opaque' && - response.status === 0 - ) { + if (request.responseTainting === 'opaque' && response.status === 0) { processBodyError(response.error) return } @@ -618,7 +628,10 @@ async function httpFetch (fetchParams) { // 3. Set response and actualResponse to the result of running // HTTP-network-or-cache fetch given fetchParams. - actualResponse = response = await httpNetworkOrCacheFetch.call(this, fetchParams) + actualResponse = response = await httpNetworkOrCacheFetch.call( + this, + fetchParams + ) // 4. If request’s response tainting is "cors" and a CORS check // for request and response returns failure, then return a network error. @@ -685,7 +698,10 @@ async function httpRedirectFetch (fetchParams, response) { let locationURL try { - locationURL = responseLocationURL(actualResponse, requestCurrentURL(request).hash) + locationURL = responseLocationURL( + actualResponse, + requestCurrentURL(request).hash + ) // 4. If locationURL is null, then return response. if (locationURL == null) { @@ -713,10 +729,7 @@ async function httpRedirectFetch (fetchParams, response) { // 9. If request’s mode is "cors", locationURL includes credentials, and // request’s origin is not same origin with locationURL’s origin, then return // a network error. - if ( - request.mode === 'cors' && - request.origin !== locationURL.origin - ) { + if (request.mode === 'cors' && request.origin !== locationURL.origin) { return makeNetworkError('cross origin not allowed for request mode "cors"') } @@ -726,7 +739,9 @@ async function httpRedirectFetch (fetchParams, response) { request.responseTainting === 'cors' && (locationURL.username || locationURL.password) ) { - return makeNetworkError('URL cannot contain credentials for request mode "cors"') + return makeNetworkError( + 'URL cannot contain credentials for request mode "cors"' + ) } // 11. If actualResponse’s status is not 303, request’s body is non-null, @@ -754,7 +769,8 @@ async function httpRedirectFetch (fetchParams, response) { // - actualResponse’s status is 303 and request’s method is not `GET` or `HEAD` if ( ([301, 302].includes(actualResponse.status) && request.method === 'POST') || - (actualResponse.status === 303 && !['GET', 'HEADER'].includes(request.method)) + (actualResponse.status === 303 && + !['GET', 'HEADER'].includes(request.method)) ) { // then: // 1. Set request’s method to `GET` and request’s body to null. @@ -781,7 +797,8 @@ async function httpRedirectFetch (fetchParams, response) { // 16. Set timingInfo’s redirect end time and post-redirect start time to the // coarsened shared current time given fetchParams’s cross-origin isolated capability. // TODO: given fetchParams’s cross-origin isolated capability? - timingInfo.redirectEndTime = timingInfo.postRedirectStartTime = performance.now() + timingInfo.redirectEndTime = timingInfo.postRedirectStartTime = + performance.now() // 17. If timingInfo’s redirect start time is 0, then set timingInfo’s redirect start // time to timingInfo’s start time. @@ -848,23 +865,24 @@ async function httpNetworkOrCacheFetch ( } // 3. Let includeCredentials be true if one of - const includeCredentials = ( + const includeCredentials = request.credentials === 'include' || - (request.credentials === 'same-origin' && request.responseTainting === 'basic') - ) + (request.credentials === 'same-origin' && + request.responseTainting === 'basic') // 4. Let contentLength be httpRequest’s body’s length, if httpRequest’s // body is non-null; otherwise null. - const contentLength = httpRequest.body - ? httpRequest.body.length - : null + const contentLength = httpRequest.body ? httpRequest.body.length : null // 5. Let contentLengthHeaderValue be null. let contentLengthHeaderValue = null // 6. If httpRequest’s body is null and httpRequest’s method is `POST` or // `PUT`, then set contentLengthHeaderValue to `0`. - if (httpRequest.body == null && ['POST', 'PUT'].includes(httpRequest.method)) { + if ( + httpRequest.body == null && + ['POST', 'PUT'].includes(httpRequest.method) + ) { contentLengthHeaderValue = '0' } @@ -936,13 +954,14 @@ async function httpNetworkOrCacheFetch ( // list contains `If-Modified-Since`, `If-None-Match`, // `If-Unmodified-Since`, `If-Match`, or `If-Range`, then set // httpRequest’s cache mode to "no-store". - if (httpRequest.cache === 'default' && ( - httpRequest.headersList.has('if-modified-since') || - httpRequest.headersList.has('if-none-match') || - httpRequest.headersList.has('if-unmodified-since') || - httpRequest.headersList.has('if-match') || - httpRequest.headersList.has('if-range') - )) { + if ( + httpRequest.cache === 'default' && + (httpRequest.headersList.has('if-modified-since') || + httpRequest.headersList.has('if-none-match') || + httpRequest.headersList.has('if-unmodified-since') || + httpRequest.headersList.has('if-match') || + httpRequest.headersList.has('if-range')) + ) { httpRequest.cache = 'no-store' } @@ -988,7 +1007,6 @@ async function httpNetworkOrCacheFetch ( // 1. If the user agent is not configured to block cookies for httpRequest // (see section 7 of [COOKIES]), then: // TODO - // 2. If httpRequest’s header list does not contain `Authorization`, then: // TODO } @@ -1038,7 +1056,8 @@ async function httpNetworkOrCacheFetch ( // Caching, and set storedResponse to null. [HTTP-CACHING] if ( !safeMethods.includes(httpRequest.method) && - (forwardResponse.status >= 200 && forwardResponse.status) + forwardResponse.status >= 200 && + forwardResponse.status ) { // TODO } @@ -1088,7 +1107,9 @@ async function httpNetworkOrCacheFetch ( // 1. Let aborted be the termination’s aborted flag. // 2. If aborted is set, then return an aborted network error. // 3. Return a network error. - return makeNetworkError(context.terminated.aborted ? new AbortError() : null) + return makeNetworkError( + context.terminated.aborted ? new AbortError() : null + ) } // 4. Prompt the end user as appropriate in request’s window and store @@ -1183,101 +1204,112 @@ function httpNetworkFetch ( assert(!context.controller) const url = requestCurrentURL(request) - return new Promise((resolve) => context.dispatcher.dispatch({ - path: url.pathname + url.search, - origin: url.origin, - method: request.method, - body: request.body ? request.body.stream : request.body, - headers: request.headersList, - maxRedirections: 0 - }, { - onConnect (abort) { - if (context.terminated) { - abort(new AbortError()) - } else { - context.abort = (err) => abort(err ?? new AbortError()) - } - }, - - onHeaders (status, headersList, resume, statusText) { - if (status < 200) { - return - } - - const headers = new Headers() - for (let n = 0; n < headersList.length; n += 2) { - headers.append(headersList[n + 0].toString(), headersList[n + 1].toString()) - } - - const stream = status === 204 - ? null - : new ReadableStream({ - async start (controller) { - context.controller = controller - }, - async pull () { - resume() - }, - async cancel (reason) { - let err - if (reason instanceof Error) { - err = reason - } else if (typeof reason === 'string') { - err = new Error(reason) - } else { - err = new AbortError() - } - - context.abort(err) + return new Promise((resolve) => + context.dispatcher.dispatch( + { + path: url.pathname + url.search, + origin: url.origin, + method: request.method, + body: request.body ? request.body.stream : request.body, + headers: request.headersList, + maxRedirections: 0 + }, + { + onConnect (abort) { + if (context.terminated) { + abort(new AbortError()) + } else { + context.abort = (err) => abort(err ?? new AbortError()) } - }, { highWaterMark: 16384 }) - - response = makeResponse({ - status, - statusText, - headersList: - headers[kHeadersList], - body: stream ? { stream } : null - }) - - resolve(response) - - return false - }, - - onData (chunk) { - assert(context.controller) + }, - // Copy the Buffer to detach it from Buffer pool. - // TODO: Is this required? - chunk = new Uint8Array(chunk) - - context.controller.enqueue(chunk) - - return context.controller.desiredSize > 0 - }, - - onComplete () { - assert(context.controller) - - context.controller.close() - context.controller = null - context.abort = null + onHeaders (status, headersList, resume, statusText) { + if (status < 200) { + return + } - finalizeResponse(fetchParams, response) - }, + const headers = new Headers() + for (let n = 0; n < headersList.length; n += 2) { + headers.append( + headersList[n + 0].toString(), + headersList[n + 1].toString() + ) + } - onError (err) { - if (context.controller) { - context.controller.error(err) - context.controller = null - context.abort = null - } else { - // TODO: What if 204? - resolve(makeNetworkError(err)) + const stream = + status === 204 + ? null + : new ReadableStream( + { + async start (controller) { + context.controller = controller + }, + async pull () { + resume() + }, + async cancel (reason) { + let err + if (reason instanceof Error) { + err = reason + } else if (typeof reason === 'string') { + err = new Error(reason) + } else { + err = new AbortError() + } + + context.abort(err) + } + }, + { highWaterMark: 16384 } + ) + + response = makeResponse({ + status, + statusText, + headersList: headers[kHeadersList], + body: stream ? { stream } : null + }) + + resolve(response) + + return false + }, + + onData (chunk) { + assert(context.controller) + + // Copy the Buffer to detach it from Buffer pool. + // TODO: Is this required? + chunk = new Uint8Array(chunk) + + context.controller.enqueue(chunk) + + return context.controller.desiredSize > 0 + }, + + onComplete () { + assert(context.controller) + + context.controller.close() + context.controller = null + context.abort = null + + finalizeResponse(fetchParams, response) + }, + + onError (err) { + if (context.controller) { + context.controller.error(err) + context.controller = null + context.abort = null + } else { + // TODO: What if 204? + resolve(makeNetworkError(err)) + } + } } - } - })) + ) + ) } function createDeferredPromise () { diff --git a/lib/api/api-fetch/request.js b/lib/api/api-fetch/request.js index d19ae663361..50f68c61b27 100644 --- a/lib/api/api-fetch/request.js +++ b/lib/api/api-fetch/request.js @@ -16,12 +16,7 @@ const { requestCache } = require('./constants') const { kEnumerableProperty } = util -const { - kHeaders, - kSignal, - kState, - kGuard -} = require('./symbols') +const { kHeaders, kSignal, kState, kGuard } = require('./symbols') const { kHeadersList } = require('../../core/symbols') const assert = require('assert') @@ -63,7 +58,10 @@ class Request { // 3. If parsedURL includes credentials, then throw a TypeError. if (parsedURL.username || parsedURL.password) { - throw new TypeError('Request cannot be constructed from a URL that includes credentials: ' + input) + throw new TypeError( + 'Request cannot be constructed from a URL that includes credentials: ' + + input + ) } // 4. Set request to a new request whose URL is parsedURL. @@ -143,7 +141,9 @@ class Request { try { parsedReferrer = new URL(referrer, baseUrl) } catch (err) { - const error = new TypeError(`Referrer "${referrer}" is not a valid URL.`) + const error = new TypeError( + `Referrer "${referrer}" is not a valid URL.` + ) error.cause = err throw error } @@ -165,7 +165,9 @@ class Request { if ('referrerPolicy' in init) { request.referrerPolicy = init.referrerPolicy if (!referrerPolicy.includes(request.referrerPolicy)) { - throw new TypeError(`Failed to construct 'Request': The provided value '${request.referrerPolicy}' is not a valid enum value of type ReferrerPolicy.`) + throw new TypeError( + `Failed to construct 'Request': The provided value '${request.referrerPolicy}' is not a valid enum value of type ReferrerPolicy.` + ) } } @@ -174,7 +176,9 @@ class Request { if ('mode' in init) { mode = init.mode if (!requestMode.includes(mode)) { - throw new TypeError(`Failed to construct 'Request': The provided value '${request.mode}' is not a valid enum value of type RequestMode.`) + throw new TypeError( + `Failed to construct 'Request': The provided value '${request.mode}' is not a valid enum value of type RequestMode.` + ) } } else { mode = fallbackMode @@ -195,7 +199,9 @@ class Request { if ('credentials' in init) { request.credentials = init.credentials if (!requestCredentials.includes(request.credentials)) { - throw new TypeError(`Failed to construct 'Request': The provided value '${request.credentials}' is not a valid enum value of type RequestCredentials.`) + throw new TypeError( + `Failed to construct 'Request': The provided value '${request.credentials}' is not a valid enum value of type RequestCredentials.` + ) } } @@ -203,21 +209,27 @@ class Request { if ('cache' in init) { request.cache = init.cache if (!requestCache.includes(request.cache)) { - throw new TypeError(`Failed to construct 'Request': The provided value '${request.cache}' is not a valid enum value of type RequestCache.`) + throw new TypeError( + `Failed to construct 'Request': The provided value '${request.cache}' is not a valid enum value of type RequestCache.` + ) } } // 21. If request’s cache mode is "only-if-cached" and request’s mode is // not "same-origin", then throw a TypeError. if (request.cache === 'only-if-cached' && request.mode !== 'same-origin') { - throw new TypeError('\'only-if-cached\' can be set only with \'same-origin\' mode') + throw new TypeError( + "'only-if-cached' can be set only with 'same-origin' mode" + ) } // 22. If init["redirect"] exists, then set request’s redirect mode to it. if ('redirect' in init) { request.redirect = init.redirect if (!requestRedirect.includes(request.redirect)) { - throw new TypeError(`Failed to construct 'Request': The provided value '${request.redirect}' is not a valid enum value of type RequestRedirect.`) + throw new TypeError( + `Failed to construct 'Request': The provided value '${request.redirect}' is not a valid enum value of type RequestRedirect.` + ) } } @@ -273,17 +285,24 @@ class Request { if ( !signal || typeof signal.aborted !== 'boolean' || - typeof signal.addEventListener !== 'function') { - throw new TypeError('Failed to construct \'Request\': member signal is not of type AbortSignal.') + typeof signal.addEventListener !== 'function' + ) { + throw new TypeError( + "Failed to construct 'Request': member signal is not of type AbortSignal." + ) } if (signal.aborted) { ac.abort() } else { // TODO: Remove this listener on failure/success. - signal.addEventListener('abort', function () { - ac.abort() - }, { once: true }) + signal.addEventListener( + 'abort', + function () { + ac.abort() + }, + { once: true } + ) } } @@ -300,7 +319,9 @@ class Request { // 1. If this’s request’s method is not a CORS-safelisted method, // then throw a TypeError. if (!corsSafeListedMethods.includes(request.method)) { - throw new TypeError(`'${request.method} is unsupported in no-cors mode.`) + throw new TypeError( + `'${request.method} is unsupported in no-cors mode.` + ) } // 2. Set this’s headers’s guard to "request-no-cors". @@ -354,7 +375,10 @@ class Request { // 1. Let Content-Type be null. // 2. Set initBody and Content-Type to the result of extracting // init["body"], with keepalive set to request’s keepalive. - const [extractedBody, contentType] = extractBody(init.body, request.keepalive) + const [extractedBody, contentType] = extractBody( + init.body, + request.keepalive + ) initBody = extractedBody // 3, If Content-Type is non-null and this’s headers’s header list does @@ -375,7 +399,9 @@ class Request { // 1. If this’s request’s mode is neither "same-origin" nor "cors", // then throw a TypeError. if (request.mode !== 'same-origin' && request.mode !== 'cors') { - throw new TypeError('If request is made from ReadableStream, mode should be "same-origin" or "cors"') + throw new TypeError( + 'If request is made from ReadableStream, mode should be "same-origin" or "cors"' + ) } // 2. Set this’s request’s use-CORS-preflight flag. @@ -389,7 +415,9 @@ class Request { if (initBody == null && inputBody !== null) { // 1. If input is unusable, then throw a TypeError. if (util.isDisturbed(inputBody.stream) || inputBody.stream.locked) { - throw new TypeError('Cannot construct a Request with a Request object that has already been used.') + throw new TypeError( + 'Cannot construct a Request with a Request object that has already been used.' + ) } // 2. Set finalBody to the result of creating a proxy for inputBody. @@ -568,9 +596,13 @@ class Request { if (this.signal.aborted) { ac.abort() } else { - this.signal.addEventListener('abort', function () { - ac.abort() - }, { once: true }) + this.signal.addEventListener( + 'abort', + function () { + ac.abort() + }, + { once: true } + ) } clonedRequestObject[kSignal] = ac.signal @@ -623,9 +655,7 @@ function makeRequest (init) { headersList: init.headersList ? new HeadersList(...init.headersList) : new HeadersList(), - urlList: init.urlList - ? [...init.urlList.map(url => new URL(url))] - : [] + urlList: init.urlList ? [...init.urlList.map((url) => new URL(url))] : [] } request.url = request.urlList[0] return request diff --git a/lib/api/api-fetch/response.js b/lib/api/api-fetch/response.js index 6d3fad9dbb7..017d82434d6 100644 --- a/lib/api/api-fetch/response.js +++ b/lib/api/api-fetch/response.js @@ -5,12 +5,12 @@ const { extractBody, cloneBody, mixinBody } = require('./body') const util = require('../../core/util') const { kEnumerableProperty } = util const { responseURL, isValidReasonPhrase } = require('./util') -const { redirectStatus, nullBodyStatus, forbiddenHeaderNames } = require('./constants') const { - kState, - kHeaders, - kGuard -} = require('./symbols') + redirectStatus, + nullBodyStatus, + forbiddenHeaderNames +} = require('./constants') +const { kState, kHeaders, kGuard } = require('./symbols') const { kHeadersList } = require('../../core/symbols') const assert = require('assert') @@ -79,7 +79,9 @@ class Response { } if (init.status < 200 || init.status > 599) { - throw new RangeError(`Failed to construct 'Response': The status provided (${init.status}) is outside the range [200, 599].`) + throw new RangeError( + `Failed to construct 'Response': The status provided (${init.status}) is outside the range [200, 599].` + ) } } @@ -245,7 +247,10 @@ function cloneResponse (response) { // filtered response whose internal response is a clone of response’s // internal response. if (response.internalResponse) { - return filterResponse(cloneResponse(response.internalResponse), response.type) + return filterResponse( + cloneResponse(response.internalResponse), + response.type + ) } // 2. Let newResponse be a copy of response, except for its body. @@ -275,9 +280,7 @@ function makeResponse (init) { headersList: init.headersList ? new HeadersList(...init.headersList) : new HeadersList(), - urlList: init.urlList - ? [...init.urlList] - : [] + urlList: init.urlList ? [...init.urlList] : [] } } @@ -285,9 +288,10 @@ function makeNetworkError (reason) { return makeResponse({ type: 'error', status: 0, - error: reason instanceof Error - ? reason - : new Error(reason ? String(reason) : reason), + error: + reason instanceof Error + ? reason + : new Error(reason ? String(reason) : reason), aborted: reason && reason.name === 'AbortError' }) } diff --git a/lib/api/api-fetch/util.js b/lib/api/api-fetch/util.js index 849972c734c..173c177d0bb 100644 --- a/lib/api/api-fetch/util.js +++ b/lib/api/api-fetch/util.js @@ -9,12 +9,11 @@ let ReadableStream let CountQueuingStrategy const badPorts = [ - 1, 7, 9, 11, 13, 15, 17, 19, 20, 21, 22, 23, 25, 37, 42, 43, - 53, 69, 77, 79, 87, 95, 101, 102, 103, 104, 109, 110, 111, 113, - 115, 117, 119, 123, 135, 137, 139, 143, 161, 179, 389, 427, 465, - 512, 513, 514, 515, 526, 530, 531, 532, 540, 548, 554, 556, 563, - 587, 601, 636, 989, 990, 993, 995, 1719, 1720, 1723, 2049, 3659, - 4045, 5060, 5061, 6000, 6566, 6665, 6666, 6667, 6668, 6669, 6697, + 1, 7, 9, 11, 13, 15, 17, 19, 20, 21, 22, 23, 25, 37, 42, 43, 53, 69, 77, 79, + 87, 95, 101, 102, 103, 104, 109, 110, 111, 113, 115, 117, 119, 123, 135, 137, + 139, 143, 161, 179, 389, 427, 465, 512, 513, 514, 515, 526, 530, 531, 532, + 540, 548, 554, 556, 563, 587, 601, 636, 989, 990, 993, 995, 1719, 1720, 1723, + 2049, 3659, 4045, 5060, 5061, 6000, 6566, 6665, 6666, 6667, 6668, 6669, 6697, 10080 ] @@ -79,11 +78,15 @@ function requestBadPort (request) { function isValidReasonPhrase (statusText) { for (let i = 0; i < statusText.length; ++i) { const c = statusText.charCodeAt(i) - if (!( - c === 0x09 || // HTAB - (c >= 0x20 && c <= 0x7E) || // SP / VCHAR - (c >= 0x80 && c <= 0xFF) // obs-text - )) { + if ( + !( + ( + c === 0x09 || // HTAB + (c >= 0x20 && c <= 0x7e) || // SP / VCHAR + (c >= 0x80 && c <= 0xff) + ) // obs-text + ) + ) { return false } } @@ -92,10 +95,25 @@ function isValidReasonPhrase (statusText) { function isTokenChar (c) { return !( - c >= 0x7F || c <= 0x20 || c === '(' || c === ')' || c === '<' || - c === '>' || c === '@' || c === ',' || c === ';' || c === ':' || - c === '\\' || c === '"' || c === '/' || c === '[' || c === ']' || - c === '?' || c === '=' || c === '{' || c === '}' + c >= 0x7f || + c <= 0x20 || + c === '(' || + c === ')' || + c === '<' || + c === '>' || + c === '@' || + c === ',' || + c === ';' || + c === ':' || + c === '\\' || + c === '"' || + c === '/' || + c === '[' || + c === ']' || + c === '?' || + c === '=' || + c === '{' || + c === '}' ) } @@ -107,7 +125,7 @@ function isValidHTTPToken (characters) { } for (let i = 0; i < characters.length; ++i) { const c = characters.charCodeAt(i) - if (c > 0x7F || !isTokenChar(c)) { + if (c > 0x7f || !isTokenChar(c)) { return false } } @@ -165,19 +183,22 @@ function toWebReadable (streamReadable) { streamReadable.on('data', onData) - return new ReadableStream({ - start (c) { - controller = c - }, + return new ReadableStream( + { + start (c) { + controller = c + }, - pull () { - streamReadable.resume() - }, + pull () { + streamReadable.resume() + }, - cancel (reason) { - destroy(streamReadable, reason) - } - }, strategy) + cancel (reason) { + destroy(streamReadable, reason) + } + }, + strategy + ) } module.exports = { From f71abfece9729b7cd6faa81fa351aa238280bec5 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 10:48:37 +0200 Subject: [PATCH 43/56] fixup: bad port + missing cond --- lib/api/api-fetch/index.js | 2 +- lib/api/api-fetch/util.js | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/api/api-fetch/index.js b/lib/api/api-fetch/index.js index 7071369c683..26cbe620f51 100644 --- a/lib/api/api-fetch/index.js +++ b/lib/api/api-fetch/index.js @@ -1057,7 +1057,7 @@ async function httpNetworkOrCacheFetch ( if ( !safeMethods.includes(httpRequest.method) && forwardResponse.status >= 200 && - forwardResponse.status + forwardResponse.status <= 399 ) { // TODO } diff --git a/lib/api/api-fetch/util.js b/lib/api/api-fetch/util.js index 173c177d0bb..baead61348f 100644 --- a/lib/api/api-fetch/util.js +++ b/lib/api/api-fetch/util.js @@ -8,6 +8,7 @@ const { finished } = require('stream') let ReadableStream let CountQueuingStrategy +// https://fetch.spec.whatwg.org/#block-bad-port const badPorts = [ 1, 7, 9, 11, 13, 15, 17, 19, 20, 21, 22, 23, 25, 37, 42, 43, 53, 69, 77, 79, 87, 95, 101, 102, 103, 104, 109, 110, 111, 113, 115, 117, 119, 123, 135, 137, From 41eae28e9ecdd4e4d71ab3dc452e44762ee764b8 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 10:50:37 +0200 Subject: [PATCH 44/56] fixup: docs --- README.md | 39 +-------------------------------------- 1 file changed, 1 insertion(+), 38 deletions(-) diff --git a/README.md b/README.md index 727ffe419b4..017374cef02 100644 --- a/README.md +++ b/README.md @@ -159,51 +159,14 @@ https://fetch.spec.whatwg.org/ ### `undici.fetch(input[, init]): Promise` -Implements [fetch](https://fetch.spec.whatwg.org/). +https://fetch.spec.whatwg.org/#fetch-method Only supported on Node 16+. This is [experimental](https://nodejs.org/api/documentation.html#documentation_stability_index) and is not yet fully compliant the Fetch Standard. We plan to ship breaking changes to this feature until it is out of experimental. -Arguments: - -* **input** `string | Request` -* **init** `RequestInit` - Returns: `Promise` -#### Parameter: `RequestInit` - -https://fetch.spec.whatwg.org/#request-class - -* **method** `string` -* **headers** `HeadersInit` -* **body** `BodyInit?` -* **referrer** *not supported* -* **referrerPolicy** *not supported* -* **mode** *not supported* -* **credentials** *not supported* -* **cache** *not supported* -* **redirect** `RequestRedirect` -* **integrity** *not supported* -* **keepalive** *not supported* -* **signal** `AbortSignal?` -* **window** `null` - -#### Parameter: `Response` - -https://fetch.spec.whatwg.org/#response-class - -* **type** `ResponseType` -* **url** `string` -* **redirected** `boolean` -* **status** `number` -* **ok** `boolean` -* **statusText** `string` -* **headers** `Headers` - -See [Dispatcher.fetch](docs/api/Dispatcher.md#dispatcherfetchoptions-callback) for more details. - ### `undici.upgrade([url, options]): Promise` Upgrade to a different protocol. See [MDN - HTTP - Protocol upgrade mechanism](https://developer.mozilla.org/en-US/docs/Web/HTTP/Protocol_upgrade_mechanism) for more details. From 2cf729312b5d4e4285f513d9e2b871e0ff410cf6 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 10:55:31 +0200 Subject: [PATCH 45/56] fixup: move fetch to new folder --- index.js | 13 ++++++++----- lib/api/index.js | 9 --------- lib/{api/api-fetch => fetch}/LICENSE | 0 lib/{api/api-fetch => fetch}/body.js | 6 +++--- lib/{api/api-fetch => fetch}/constants.js | 0 lib/{api/api-fetch => fetch}/headers.js | 6 +++--- lib/{api/api-fetch => fetch}/index.js | 4 ++-- lib/{api/api-fetch => fetch}/request.js | 4 ++-- lib/{api/api-fetch => fetch}/response.js | 4 ++-- lib/{api/api-fetch => fetch}/symbols.js | 0 lib/{api/api-fetch => fetch}/util.js | 4 ++-- test/node-fetch/headers.js | 2 +- test/node-fetch/main.js | 6 +++--- test/node-fetch/request.js | 2 +- test/node-fetch/response.js | 4 ++-- 15 files changed, 29 insertions(+), 35 deletions(-) rename lib/{api/api-fetch => fetch}/LICENSE (100%) rename lib/{api/api-fetch => fetch}/body.js (98%) rename lib/{api/api-fetch => fetch}/constants.js (100%) rename lib/{api/api-fetch => fetch}/headers.js (98%) rename lib/{api/api-fetch => fetch}/index.js (99%) rename lib/{api/api-fetch => fetch}/request.js (99%) rename lib/{api/api-fetch => fetch}/response.js (99%) rename lib/{api/api-fetch => fetch}/symbols.js (100%) rename lib/{api/api-fetch => fetch}/util.js (97%) diff --git a/index.js b/index.js index d2f7c1e9fc4..f218bd282c3 100644 --- a/index.js +++ b/index.js @@ -14,6 +14,8 @@ const MockAgent = require('./lib/mock/mock-agent') const MockPool = require('./lib/mock/mock-pool') const mockErrors = require('./lib/mock/mock-errors') +const nodeMajor = Number(process.versions.node.split('.')[0]) + Object.assign(Dispatcher.prototype, api) module.exports.Dispatcher = Dispatcher @@ -84,14 +86,15 @@ function makeDispatcher (fn) { module.exports.setGlobalDispatcher = setGlobalDispatcher module.exports.getGlobalDispatcher = getGlobalDispatcher -if (api.fetch) { +if (nodeMajor >= 16) { + const fetchImpl = require('./lib/fetch') module.exports.fetch = async function fetch (resource, init) { const dispatcher = getGlobalDispatcher() - return api.fetch.call(dispatcher, resource, init) + return fetchImpl.call(dispatcher, resource, init) } - module.exports.Headers = api.fetch.Headers - module.exports.Response = api.fetch.Response - module.exports.Request = api.fetch.Request + module.exports.Headers = require('./lib/fetch/headers').Headers + module.exports.Response = require('./lib/fetch/response').Response + module.exports.Request = require('./lib/fetch/request').Request } module.exports.request = makeDispatcher(api.request) diff --git a/lib/api/index.js b/lib/api/index.js index 046f5c3bed4..8983a5e746f 100644 --- a/lib/api/index.js +++ b/lib/api/index.js @@ -1,16 +1,7 @@ 'use strict' -const nodeMajor = Number(process.versions.node.split('.')[0]) - module.exports.request = require('./api-request') module.exports.stream = require('./api-stream') module.exports.pipeline = require('./api-pipeline') module.exports.upgrade = require('./api-upgrade') module.exports.connect = require('./api-connect') - -if (nodeMajor >= 16) { - module.exports.fetch = require('./api-fetch') - module.exports.fetch.Headers = require('./api-fetch/headers').Headers - module.exports.fetch.Response = require('./api-fetch/response').Response - module.exports.fetch.Request = require('./api-fetch/request').Request -} diff --git a/lib/api/api-fetch/LICENSE b/lib/fetch/LICENSE similarity index 100% rename from lib/api/api-fetch/LICENSE rename to lib/fetch/LICENSE diff --git a/lib/api/api-fetch/body.js b/lib/fetch/body.js similarity index 98% rename from lib/api/api-fetch/body.js rename to lib/fetch/body.js index 0b4bc1b1d24..7e26a801f98 100644 --- a/lib/api/api-fetch/body.js +++ b/lib/fetch/body.js @@ -1,13 +1,13 @@ 'use strict' -const util = require('../../core/util') +const util = require('../core/util') const { toWebReadable } = require('./util') const { kState } = require('./symbols') const { Blob } = require('buffer') const { Readable } = require('stream') -const { NotSupportedError } = require('../../core/errors') +const { NotSupportedError } = require('../core/errors') const { ReadableStream } = require('stream/web') -const { kBodyUsed } = require('../../core/symbols') +const { kBodyUsed } = require('../core/symbols') const assert = require('assert') const nodeUtil = require('util') diff --git a/lib/api/api-fetch/constants.js b/lib/fetch/constants.js similarity index 100% rename from lib/api/api-fetch/constants.js rename to lib/fetch/constants.js diff --git a/lib/api/api-fetch/headers.js b/lib/fetch/headers.js similarity index 98% rename from lib/api/api-fetch/headers.js rename to lib/fetch/headers.js index abd50e76e45..834473b9069 100644 --- a/lib/api/api-fetch/headers.js +++ b/lib/fetch/headers.js @@ -3,14 +3,14 @@ 'use strict' const { validateHeaderName, validateHeaderValue } = require('http') -const { kHeadersList } = require('../../core/symbols') +const { kHeadersList } = require('../core/symbols') const { kGuard } = require('./symbols') -const { kEnumerableProperty } = require('../../core/util') +const { kEnumerableProperty } = require('../core/util') const { InvalidHTTPTokenError, HTTPInvalidHeaderValueError, InvalidThisError -} = require('../../core/errors') +} = require('../core/errors') const { forbiddenHeaderNames, forbiddenResponseHeaderNames diff --git a/lib/api/api-fetch/index.js b/lib/fetch/index.js similarity index 99% rename from lib/api/api-fetch/index.js rename to lib/fetch/index.js index 26cbe620f51..5b5d3e438c0 100644 --- a/lib/api/api-fetch/index.js +++ b/lib/fetch/index.js @@ -16,7 +16,7 @@ const { requestCurrentURL } = require('./util') const { kState, kHeaders, kGuard } = require('./symbols') -const { AbortError } = require('../../core/errors') +const { AbortError } = require('../core/errors') const assert = require('assert') const { safelyExtractBody, isBodyReadable } = require('./body') const { @@ -25,7 +25,7 @@ const { safeMethods, requestBodyHeader } = require('./constants') -const { kHeadersList } = require('../../core/symbols') +const { kHeadersList } = require('../core/symbols') const { ReadableStream } = require('stream/web') const { performance } = require('perf_hooks') diff --git a/lib/api/api-fetch/request.js b/lib/fetch/request.js similarity index 99% rename from lib/api/api-fetch/request.js rename to lib/fetch/request.js index 50f68c61b27..4d0d869fec5 100644 --- a/lib/api/api-fetch/request.js +++ b/lib/fetch/request.js @@ -4,7 +4,7 @@ const { extractBody, mixinBody, cloneBody } = require('./body') const { Headers, fill: fillHeaders, HeadersList } = require('./headers') -const util = require('../../core/util') +const util = require('../core/util') const { isValidHTTPToken } = require('./util') const { forbiddenMethods, @@ -17,7 +17,7 @@ const { } = require('./constants') const { kEnumerableProperty } = util const { kHeaders, kSignal, kState, kGuard } = require('./symbols') -const { kHeadersList } = require('../../core/symbols') +const { kHeadersList } = require('../core/symbols') const assert = require('assert') let TransformStream diff --git a/lib/api/api-fetch/response.js b/lib/fetch/response.js similarity index 99% rename from lib/api/api-fetch/response.js rename to lib/fetch/response.js index 017d82434d6..0912d0ad28c 100644 --- a/lib/api/api-fetch/response.js +++ b/lib/fetch/response.js @@ -2,7 +2,7 @@ const { Headers, HeadersList, fill } = require('./headers') const { extractBody, cloneBody, mixinBody } = require('./body') -const util = require('../../core/util') +const util = require('../core/util') const { kEnumerableProperty } = util const { responseURL, isValidReasonPhrase } = require('./util') const { @@ -11,7 +11,7 @@ const { forbiddenHeaderNames } = require('./constants') const { kState, kHeaders, kGuard } = require('./symbols') -const { kHeadersList } = require('../../core/symbols') +const { kHeadersList } = require('../core/symbols') const assert = require('assert') // https://fetch.spec.whatwg.org/#response-class diff --git a/lib/api/api-fetch/symbols.js b/lib/fetch/symbols.js similarity index 100% rename from lib/api/api-fetch/symbols.js rename to lib/fetch/symbols.js diff --git a/lib/api/api-fetch/util.js b/lib/fetch/util.js similarity index 97% rename from lib/api/api-fetch/util.js rename to lib/fetch/util.js index baead61348f..3863830f9b9 100644 --- a/lib/api/api-fetch/util.js +++ b/lib/fetch/util.js @@ -1,8 +1,8 @@ 'use strict' const { redirectStatus } = require('./constants') -const { destroy, isDestroyed } = require('../../../lib/core/util') -const { AbortError } = require('../../../lib/core/errors') +const { destroy, isDestroyed } = require('../../lib/core/util') +const { AbortError } = require('../../lib/core/errors') const { finished } = require('stream') let ReadableStream diff --git a/test/node-fetch/headers.js b/test/node-fetch/headers.js index 0a9212cf85a..e509fd8d893 100644 --- a/test/node-fetch/headers.js +++ b/test/node-fetch/headers.js @@ -3,7 +3,7 @@ const { format } = require('util') const chai = require('chai') const chaiIterator = require('chai-iterator') -const { Headers } = require('../../lib/api/api-fetch/headers.js') +const { Headers } = require('../../lib/fetch/headers.js') chai.use(chaiIterator) diff --git a/test/node-fetch/main.js b/test/node-fetch/main.js index 3895093dc1d..e7ab9188e6e 100644 --- a/test/node-fetch/main.js +++ b/test/node-fetch/main.js @@ -21,9 +21,9 @@ const { setGlobalDispatcher, Agent } = require('../../index.js') -const HeadersOrig = require('../../lib/api/api-fetch/headers.js').Headers -const RequestOrig = require('../../lib/api/api-fetch/request.js').Request -const ResponseOrig = require('../../lib/api/api-fetch/response.js').Response +const HeadersOrig = require('../../lib/fetch/headers.js').Headers +const RequestOrig = require('../../lib/fetch/request.js').Request +const ResponseOrig = require('../../lib/fetch/response.js').Response const TestServer = require('./utils/server.js') const chaiTimeout = require('./utils/chai-timeout.js') const { ReadableStream } = require('stream/web') diff --git a/test/node-fetch/request.js b/test/node-fetch/request.js index ea82d6add1c..4cf075536d0 100644 --- a/test/node-fetch/request.js +++ b/test/node-fetch/request.js @@ -5,7 +5,7 @@ const AbortController = require('abort-controller') const chai = require('chai') const { Blob } = require('buffer') -const Request = require('../../lib/api/api-fetch/request.js').Request +const Request = require('../../lib/fetch/request.js').Request const TestServer = require('./utils/server.js') const { expect } = chai diff --git a/test/node-fetch/response.js b/test/node-fetch/response.js index 71bc24e4052..86153873c3a 100644 --- a/test/node-fetch/response.js +++ b/test/node-fetch/response.js @@ -2,10 +2,10 @@ const chai = require('chai') const stream = require('stream') -const { Response } = require('../../lib/api/api-fetch/response.js') +const { Response } = require('../../lib/fetch/response.js') const TestServer = require('./utils/server.js') const { Blob } = require('buffer') -const { kState } = require('../../lib/api/api-fetch/symbols.js') +const { kState } = require('../../lib/fetch/symbols.js') const { expect } = chai From 83adb4e3fad88123aceb02f259cfa29b74d58499 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 10:56:11 +0200 Subject: [PATCH 46/56] fixup: Dispatcher.fetch doesn't exist --- docs/api/Dispatcher.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/api/Dispatcher.md b/docs/api/Dispatcher.md index 0ac5bb298b2..3ee3d05dc2f 100644 --- a/docs/api/Dispatcher.md +++ b/docs/api/Dispatcher.md @@ -305,12 +305,6 @@ client.dispatch({ }) ``` -### `Dispatcher.fetch(options)` - -Only supported on Node 16+. - -This is [experimental](https://nodejs.org/api/documentation.html#documentation_stability_index) and is not yet fully compliant the Fetch Standard. We plan to ship breaking changes to this feature until it is out of experimental. - ### `Dispatcher.pipeline(options, handler)` For easy use with [stream.pipeline](https://nodejs.org/api/stream.html#stream_stream_pipeline_source_transforms_destination_callback). The `handler` argument should return a `Readable` from which the result will be read. Usually it should just return the `body` argument unless some kind of transformation needs to be performed based on e.g. `headers` or `statusCode`. The `handler` should validate the response and save any required state. If there is an error, it should be thrown. The function returns a `Duplex` which writes to the request and reads from the response. From 387919bb1b3ba5d4fe7e48f991e0084e29a5f6dd Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 10:56:57 +0200 Subject: [PATCH 47/56] fixup: link MDN docs --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 017374cef02..3fc45f6f440 100644 --- a/README.md +++ b/README.md @@ -159,6 +159,7 @@ https://fetch.spec.whatwg.org/ ### `undici.fetch(input[, init]): Promise` +https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch https://fetch.spec.whatwg.org/#fetch-method Only supported on Node 16+. From efddfc27c51d42ab2798c5d1438d3fd51debcc7f Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 11:17:43 +0200 Subject: [PATCH 48/56] fixup --- lib/fetch/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/fetch/index.js b/lib/fetch/index.js index 5b5d3e438c0..4db9d70aff2 100644 --- a/lib/fetch/index.js +++ b/lib/fetch/index.js @@ -1115,11 +1115,11 @@ async function httpNetworkOrCacheFetch ( // 4. Prompt the end user as appropriate in request’s window and store // the result as a proxy-authentication entry. [HTTP-AUTH] // TODO: Invoke some kind of callback? - return makeNetworkError('proxy authentication required') // 5. Set response to the result of running HTTP-network-or-cache fetch given // fetchParams. // TODO + return makeNetworkError('proxy authentication required') } // 15. If all of the following are true @@ -1190,7 +1190,7 @@ function httpNetworkFetch ( // partition key given request. // TODO - // ... + // TODO... const context = this From c3822cfd64581177bb412b1852e01cb9985e1040 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 11:33:48 +0200 Subject: [PATCH 49/56] fixup: node fetch in CI --- .github/workflows/nodejs.yml | 7 +++---- lib/fetch/index.js | 31 +++++++++++++++++++++++++++++++ package.json | 2 +- scripts/test-node-fetch.js | 11 +++++++++++ 4 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 scripts/test-node-fetch.js diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml index 0dd4f6f33e6..e73cc637d07 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/nodejs.yml @@ -36,10 +36,9 @@ jobs: run: | npm run test:jest - # How to run only for Node 16+? - # - name: Test node-fetch - # run: | - # npm run test:node-fetch + - name: Test node-fetch + run: | + npm run test:node-fetch - name: Test Types run: | diff --git a/lib/fetch/index.js b/lib/fetch/index.js index 4db9d70aff2..0eca31fa186 100644 --- a/lib/fetch/index.js +++ b/lib/fetch/index.js @@ -1190,6 +1190,37 @@ function httpNetworkFetch ( // partition key given request. // TODO + // 7. Switch on request’s mode: + // TODO + + // 8. Run these steps, but abort when the ongoing fetch is terminated: + + // 5 .Set response to the result of making an HTTP request over connection + // using request with the following caveats: + + // Follow the relevant requirements from HTTP. [HTTP] [HTTP-SEMANTICS] + // [HTTP-COND] [HTTP-CACHING] [HTTP-AUTH] + + // If request’s body is non-null, and request’s body’s source is null, + // then the user agent may have a buffer of up to 64 kibibytes and store + // a part of request’s body in that buffer. If the user agent reads from + // request’s body beyond that buffer’s size and the user agent needs to + // resend request, then instead return a network error. + // TODO + + // Set timingInfo’s final network-response start time to the coarsened + // shared current time given fetchParams’s cross-origin isolated capability, + // immediately after the user agent’s HTTP parser receives the first byte + // of the response (e.g., frame header bytes for HTTP/2 or response status + // line for HTTP/1.x). + // TODO + + // Wait until all the headers are transmitted. + + // Any responses whose status is in the range 100 to 199, inclusive, + // and is not 101, are to be ignored, except for the purposes of setting + // timingInfo’s final network-response start time above. + // TODO... const context = this diff --git a/package.json b/package.json index 8689f11b2c5..268bd61ef0f 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "lint": "standard | snazzy", "lint:fix": "standard --fix | snazzy", "test": "tap test/*.js --no-coverage && mocha test/node-fetch && jest test/jest/test", - "test:node-fetch": "mocha test/node-fetch", + "test:node-fetch": "node scripts/test-node-fetch.js 16 && mocha test/node-fetch || echo Skipping", "test:jest": "jest test/jest/test", "test:tap": "tap test/*.js --no-coverage ", "test:tdd": "tap test/*.js -w --no-coverage-report", diff --git a/scripts/test-node-fetch.js b/scripts/test-node-fetch.js new file mode 100644 index 00000000000..dd6f221c86a --- /dev/null +++ b/scripts/test-node-fetch.js @@ -0,0 +1,11 @@ +const [major, minor, patch] = process.versions.node.split('.').map(v => Number(v)) +const required = process.argv.pop().split('.').map(v => Number(v)) + +const badMajor = major < required[0] +const badMinor = major === required[0] && minor < required[1] +const badPatch = major === required[0] && minor < required[1] && patch < required[2] + +if (badMajor || badMinor || badPatch) { + console.log(`Required Node.js >=${required.join('.')}, got ${process.versions.node}`) + process.exit(1) +} From abe0c685781c511bd4c0141ca2d6bf99a19c6b4c Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 11:39:40 +0200 Subject: [PATCH 50/56] fixup: terminate on onError --- lib/fetch/index.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/fetch/index.js b/lib/fetch/index.js index 0eca31fa186..c9405a2419e 100644 --- a/lib/fetch/index.js +++ b/lib/fetch/index.js @@ -1319,20 +1319,24 @@ function httpNetworkFetch ( }, onComplete () { + context.abort = null + assert(context.controller) context.controller.close() context.controller = null - context.abort = null finalizeResponse(fetchParams, response) }, onError (err) { + context.abort = null + + context.terminate({ aborted: err.name === 'AbortError' }) + if (context.controller) { context.controller.error(err) context.controller = null - context.abort = null } else { // TODO: What if 204? resolve(makeNetworkError(err)) From 32aef9fd04916eeb40155f21fe46709534b9c94d Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 12:44:21 +0200 Subject: [PATCH 51/56] fixup --- scripts/test-node-fetch.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-node-fetch.js b/scripts/test-node-fetch.js index dd6f221c86a..fbccb87cfc4 100644 --- a/scripts/test-node-fetch.js +++ b/scripts/test-node-fetch.js @@ -3,7 +3,7 @@ const required = process.argv.pop().split('.').map(v => Number(v)) const badMajor = major < required[0] const badMinor = major === required[0] && minor < required[1] -const badPatch = major === required[0] && minor < required[1] && patch < required[2] +const badPatch = major === required[0] && minor === required[1] && patch < required[2] if (badMajor || badMinor || badPatch) { console.log(`Required Node.js >=${required.join('.')}, got ${process.versions.node}`) From 6f005a6af8c80495c305388ffcd8a80f856a745b Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 12:54:58 +0200 Subject: [PATCH 52/56] fixup: transmit request body algorithm --- lib/fetch/index.js | 71 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 69 insertions(+), 2 deletions(-) diff --git a/lib/fetch/index.js b/lib/fetch/index.js index c9405a2419e..2a1d1a6cc11 100644 --- a/lib/fetch/index.js +++ b/lib/fetch/index.js @@ -1195,7 +1195,7 @@ function httpNetworkFetch ( // 8. Run these steps, but abort when the ongoing fetch is terminated: - // 5 .Set response to the result of making an HTTP request over connection + // 5. Set response to the result of making an HTTP request over connection // using request with the following caveats: // Follow the relevant requirements from HTTP. [HTTP] [HTTP-SEMANTICS] @@ -1221,6 +1221,73 @@ function httpNetworkFetch ( // and is not 101, are to be ignored, except for the purposes of setting // timingInfo’s final network-response start time above. + // If request’s header list contains `Transfer-Encoding`/`chunked` and + // response is transferred via HTTP/1.0 or older, then return a network + // error. + + // If the HTTP request results in a TLS client certificate dialog, then: + + // 1. If request’s window is an environment settings object, make the + // dialog available in request’s window. + + // 2. Otherwise, return a network error. + + // To transmit request’s body body, run these steps: + const body = (async function * () { + // 1. If body is null and fetchParams’s process request end-of-body is + // non-null, then queue a fetch task given fetchParams’s process request + // end-of-body and fetchParams’s task destination. + if (request.body === null) { + fetchParams.processEndOfBody?.() + return + } + + // 2. Otherwise, if body is non-null: + try { + // 1. Let processBodyChunk given bytes be these steps: + for await (const bytes of request.body.stream) { + // 1. If the ongoing fetch is terminated, then abort these steps. + if (context.terminated) { + return + } + + // 2. Run this step in parallel: transmit bytes. + yield bytes + + // 3. If fetchParams’s process request body is non-null, then run + // fetchParams’s process request body given bytes’s length. + fetchParams.processRequestBody?.(bytes.byteLength) + } + + // 2. Let processEndOfBody be these steps: + + // 1. If the ongoing fetch is terminated, then abort these steps. + if (context.terminated) { + return + } + + // 2. If fetchParams’s process request end-of-body is non-null, + // then run fetchParams’s process request end-of-body. + fetchParams.processRequestEndOfBody?.() + } catch (e) { + // 3. Let processBodyError given e be these steps: + + // 1. If the ongoing fetch is terminated, then abort these steps. + if (context.terminated) { + return + } + + // 2. If e is an "AbortError" DOMException, then terminate the ongoing fetch with the aborted flag set. + if (e.name === 'AbortError') { + context.terminate({ aborted: true }) + return + } + + // 3. Otherwise, terminate the ongoing fetch. + context.terminate() + } + })() + // TODO... const context = this @@ -1241,7 +1308,7 @@ function httpNetworkFetch ( path: url.pathname + url.search, origin: url.origin, method: request.method, - body: request.body ? request.body.stream : request.body, + body, headers: request.headersList, maxRedirections: 0 }, From a901f6c93234d27b7273d21ed0d0e2271dacdd2c Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 13:01:59 +0200 Subject: [PATCH 53/56] fixup --- lib/fetch/index.js | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/lib/fetch/index.js b/lib/fetch/index.js index 2a1d1a6cc11..402e3b1a591 100644 --- a/lib/fetch/index.js +++ b/lib/fetch/index.js @@ -28,10 +28,11 @@ const { const { kHeadersList } = require('../core/symbols') const { ReadableStream } = require('stream/web') const { performance } = require('perf_hooks') +const EE = require('events') // https://fetch.spec.whatwg.org/#fetch-method async function fetch (resource, init) { - const context = { + const context = Object.assign(new EE(), { dispatcher: this, abort: null, controller: null, @@ -49,8 +50,10 @@ async function fetch (resource, init) { this.abort() this.abort = null } + + this.emit('terminated') } - } + }) // 1. Let p be a new promise. const p = createDeferredPromise() @@ -1167,6 +1170,8 @@ function httpNetworkFetch ( includeCredentials = false, forceNewConnection = false ) { + const context = this + // 1. Let request be fetchParams’s request. const request = fetchParams.request @@ -1194,6 +1199,7 @@ function httpNetworkFetch ( // TODO // 8. Run these steps, but abort when the ongoing fetch is terminated: + context.on('terminated', onRequestAborted) // 5. Set response to the result of making an HTTP request over connection // using request with the following caveats: @@ -1285,12 +1291,26 @@ function httpNetworkFetch ( // 3. Otherwise, terminate the ongoing fetch. context.terminate() + } finally { + context.off('terminated', onRequestAborted) } })() - // TODO... + function onRequestAborted () { + // 1. Let aborted be the termination’s aborted flag. + // TODO - const context = this + // 2. If connection uses HTTP/2, then transmit an RST_STREAM frame. + if (context.abort) { + context.abort() + } + + // 3. If aborted is set, then return an aborted network error. + // 4. Return a network error. + // TODO + } + + // TODO... // TODO: forceNewConnection From b6cb85d6ced55da8c83a48b89dd1e2cc6c1cc93a Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 13:11:02 +0200 Subject: [PATCH 54/56] fixup --- lib/fetch/index.js | 247 +++++++++++++++++++++++---------------------- 1 file changed, 124 insertions(+), 123 deletions(-) diff --git a/lib/fetch/index.js b/lib/fetch/index.js index 402e3b1a591..a74d90e38d8 100644 --- a/lib/fetch/index.js +++ b/lib/fetch/index.js @@ -34,7 +34,6 @@ const EE = require('events') async function fetch (resource, init) { const context = Object.assign(new EE(), { dispatcher: this, - abort: null, controller: null, terminated: false, terminate ({ aborted } = {}) { @@ -42,15 +41,13 @@ async function fetch (resource, init) { return } - this.terminated = { aborted } - - assert(!this.abort || aborted) - - if (this.abort) { - this.abort() - this.abort = null + if (context.abort) { + context.abort() + context.abort = null } + this.terminated = { aborted } + this.emit('terminated') } }) @@ -517,8 +514,8 @@ async function mainFetch (fetchParams, recursive = false) { nullBodyStatus.includes(internalResponse.status)) ) { internalResponse.body = null - if (context.abort) { - context.abort() + if (context.controller) { + context.controller.error(new AbortError()) } } @@ -1170,159 +1167,163 @@ function httpNetworkFetch ( includeCredentials = false, forceNewConnection = false ) { - const context = this + return new Promise((resolve) => { + const context = this - // 1. Let request be fetchParams’s request. - const request = fetchParams.request + // 1. Let request be fetchParams’s request. + const request = fetchParams.request - // 2. Let response be null. - let response = null + // 2. Let response be null. + let response = null - // 3. Let timingInfo be fetchParams’s timing info. - // TODO + // 3. Let timingInfo be fetchParams’s timing info. + // TODO - // 4. Let httpCache be the result of determining the HTTP cache partition, - // given request. - // TODO - const httpCache = null + // 4. Let httpCache be the result of determining the HTTP cache partition, + // given request. + // TODO + const httpCache = null - // 5. If httpCache is null, then set request’s cache mode to "no-store". - if (httpCache == null) { - request.cache = 'no-store' - } + // 5. If httpCache is null, then set request’s cache mode to "no-store". + if (httpCache == null) { + request.cache = 'no-store' + } - // 6. Let networkPartitionKey be the result of determining the network - // partition key given request. - // TODO + // 6. Let networkPartitionKey be the result of determining the network + // partition key given request. + // TODO - // 7. Switch on request’s mode: - // TODO + // 7. Switch on request’s mode: + // TODO - // 8. Run these steps, but abort when the ongoing fetch is terminated: - context.on('terminated', onRequestAborted) + // 8. Run these steps, but abort when the ongoing fetch is terminated: + // TODO: When do we cleanup this listener? + context.on('terminated', onRequestAborted) - // 5. Set response to the result of making an HTTP request over connection - // using request with the following caveats: + // 5. Set response to the result of making an HTTP request over connection + // using request with the following caveats: - // Follow the relevant requirements from HTTP. [HTTP] [HTTP-SEMANTICS] - // [HTTP-COND] [HTTP-CACHING] [HTTP-AUTH] + // Follow the relevant requirements from HTTP. [HTTP] [HTTP-SEMANTICS] + // [HTTP-COND] [HTTP-CACHING] [HTTP-AUTH] - // If request’s body is non-null, and request’s body’s source is null, - // then the user agent may have a buffer of up to 64 kibibytes and store - // a part of request’s body in that buffer. If the user agent reads from - // request’s body beyond that buffer’s size and the user agent needs to - // resend request, then instead return a network error. - // TODO + // If request’s body is non-null, and request’s body’s source is null, + // then the user agent may have a buffer of up to 64 kibibytes and store + // a part of request’s body in that buffer. If the user agent reads from + // request’s body beyond that buffer’s size and the user agent needs to + // resend request, then instead return a network error. + // TODO - // Set timingInfo’s final network-response start time to the coarsened - // shared current time given fetchParams’s cross-origin isolated capability, - // immediately after the user agent’s HTTP parser receives the first byte - // of the response (e.g., frame header bytes for HTTP/2 or response status - // line for HTTP/1.x). - // TODO + // Set timingInfo’s final network-response start time to the coarsened + // shared current time given fetchParams’s cross-origin isolated capability, + // immediately after the user agent’s HTTP parser receives the first byte + // of the response (e.g., frame header bytes for HTTP/2 or response status + // line for HTTP/1.x). + // TODO - // Wait until all the headers are transmitted. + // Wait until all the headers are transmitted. - // Any responses whose status is in the range 100 to 199, inclusive, - // and is not 101, are to be ignored, except for the purposes of setting - // timingInfo’s final network-response start time above. + // Any responses whose status is in the range 100 to 199, inclusive, + // and is not 101, are to be ignored, except for the purposes of setting + // timingInfo’s final network-response start time above. - // If request’s header list contains `Transfer-Encoding`/`chunked` and - // response is transferred via HTTP/1.0 or older, then return a network - // error. + // If request’s header list contains `Transfer-Encoding`/`chunked` and + // response is transferred via HTTP/1.0 or older, then return a network + // error. - // If the HTTP request results in a TLS client certificate dialog, then: + // If the HTTP request results in a TLS client certificate dialog, then: - // 1. If request’s window is an environment settings object, make the - // dialog available in request’s window. + // 1. If request’s window is an environment settings object, make the + // dialog available in request’s window. - // 2. Otherwise, return a network error. + // 2. Otherwise, return a network error. - // To transmit request’s body body, run these steps: - const body = (async function * () { - // 1. If body is null and fetchParams’s process request end-of-body is - // non-null, then queue a fetch task given fetchParams’s process request - // end-of-body and fetchParams’s task destination. - if (request.body === null) { - fetchParams.processEndOfBody?.() - return - } + // To transmit request’s body body, run these steps: + const body = (async function * () { + try { + // 1. If body is null and fetchParams’s process request end-of-body is + // non-null, then queue a fetch task given fetchParams’s process request + // end-of-body and fetchParams’s task destination. + if (request.body === null) { + fetchParams.processEndOfBody?.() + return + } - // 2. Otherwise, if body is non-null: - try { - // 1. Let processBodyChunk given bytes be these steps: - for await (const bytes of request.body.stream) { - // 1. If the ongoing fetch is terminated, then abort these steps. + // 2. Otherwise, if body is non-null: + + // 1. Let processBodyChunk given bytes be these steps: + for await (const bytes of request.body.stream) { + // 1. If the ongoing fetch is terminated, then abort these steps. + if (context.terminated) { + return + } + + // 2. Run this step in parallel: transmit bytes. + yield bytes + + // 3. If fetchParams’s process request body is non-null, then run + // fetchParams’s process request body given bytes’s length. + fetchParams.processRequestBody?.(bytes.byteLength) + } + + // 2. Let processEndOfBody be these steps: + + // 1. If the ongoing fetch is terminated, then abort these steps. if (context.terminated) { return } - // 2. Run this step in parallel: transmit bytes. - yield bytes + // 2. If fetchParams’s process request end-of-body is non-null, + // then run fetchParams’s process request end-of-body. + fetchParams.processRequestEndOfBody?.() + } catch (e) { + // 3. Let processBodyError given e be these steps: - // 3. If fetchParams’s process request body is non-null, then run - // fetchParams’s process request body given bytes’s length. - fetchParams.processRequestBody?.(bytes.byteLength) - } + // 1. If the ongoing fetch is terminated, then abort these steps. + if (context.terminated) { + return + } - // 2. Let processEndOfBody be these steps: + // 2. If e is an "AbortError" DOMException, then terminate the ongoing fetch with the aborted flag set. + if (e.name === 'AbortError') { + context.terminate({ aborted: true }) + return + } - // 1. If the ongoing fetch is terminated, then abort these steps. - if (context.terminated) { - return + // 3. Otherwise, terminate the ongoing fetch. + context.terminate() } + })() - // 2. If fetchParams’s process request end-of-body is non-null, - // then run fetchParams’s process request end-of-body. - fetchParams.processRequestEndOfBody?.() - } catch (e) { - // 3. Let processBodyError given e be these steps: + function onRequestAborted () { + // 1. Let aborted be the termination’s aborted flag. + const aborted = context.terminated.aborted - // 1. If the ongoing fetch is terminated, then abort these steps. - if (context.terminated) { - return + // 2. If connection uses HTTP/2, then transmit an RST_STREAM frame. + if (context.abort) { + context.abort() } - // 2. If e is an "AbortError" DOMException, then terminate the ongoing fetch with the aborted flag set. - if (e.name === 'AbortError') { - context.terminate({ aborted: true }) - return + // 3. If aborted is set, then return an aborted network error. + if (aborted) { + resolve(makeNetworkError(new AbortError())) } - // 3. Otherwise, terminate the ongoing fetch. - context.terminate() - } finally { - context.off('terminated', onRequestAborted) + // 4. Return a network error. + resolve(makeNetworkError()) } - })() - function onRequestAborted () { - // 1. Let aborted be the termination’s aborted flag. - // TODO + // TODO... - // 2. If connection uses HTTP/2, then transmit an RST_STREAM frame. - if (context.abort) { + // TODO: forceNewConnection + + // NOTE: This is just a hack. + if (forceNewConnection && context.controller) { context.abort() } - // 3. If aborted is set, then return an aborted network error. - // 4. Return a network error. - // TODO - } - - // TODO... - - // TODO: forceNewConnection - - // NOTE: This is just a hack. - if (forceNewConnection && context.controller) { - context.abort() - } - - assert(!context.controller) + assert(!context.controller) - const url = requestCurrentURL(request) - return new Promise((resolve) => + const url = requestCurrentURL(request) context.dispatcher.dispatch( { path: url.pathname + url.search, @@ -1431,7 +1432,7 @@ function httpNetworkFetch ( } } ) - ) + }) } function createDeferredPromise () { From 53b3ab160b7fc40c6633c18290ee7b3bf31bcc48 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 13:32:11 +0200 Subject: [PATCH 55/56] fixup --- lib/fetch/index.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/fetch/index.js b/lib/fetch/index.js index a74d90e38d8..895b6206af3 100644 --- a/lib/fetch/index.js +++ b/lib/fetch/index.js @@ -36,6 +36,7 @@ async function fetch (resource, init) { dispatcher: this, controller: null, terminated: false, + abort: null, // "connection" abort terminate ({ aborted } = {}) { if (this.terminated) { return @@ -43,7 +44,6 @@ async function fetch (resource, init) { if (context.abort) { context.abort() - context.abort = null } this.terminated = { aborted } @@ -1407,8 +1407,6 @@ function httpNetworkFetch ( }, onComplete () { - context.abort = null - assert(context.controller) context.controller.close() @@ -1418,8 +1416,6 @@ function httpNetworkFetch ( }, onError (err) { - context.abort = null - context.terminate({ aborted: err.name === 'AbortError' }) if (context.controller) { From 260cf8f3fb6b88a5bdbc293b168263277da2bd09 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Thu, 12 Aug 2021 13:38:04 +0200 Subject: [PATCH 56/56] fixup --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3fc45f6f440..12d08482acc 100644 --- a/README.md +++ b/README.md @@ -155,10 +155,10 @@ Calls `options.dispatch.connect(options)`. See [Dispatcher.connect](docs/api/Dispatcher.md#dispatcherconnect) for more details. -https://fetch.spec.whatwg.org/ - ### `undici.fetch(input[, init]): Promise` +Implements [fetch](https://fetch.spec.whatwg.org/). + https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch https://fetch.spec.whatwg.org/#fetch-method