Merge branch 'master' into gitlab-release

core/base-service/base.js

@@ -20,7 +20,7 @@ import {
   Deprecated,
 } from './errors.js'
 import { validateExample, transformExample } from './examples.js'
-import { sendRequest } from './node-fetch.js'
+import { fetchFactory } from './got.js'
 import {
   makeFullUrl,
   assertValidRoute,
@@ -432,7 +432,7 @@ class BaseService {
       ServiceClass: this,
     })
 
-    const fetcher = sendRequest.bind(sendRequest, fetchLimitBytes)
+    const fetcher = fetchFactory(fetchLimitBytes)
 
     camp.route(
       regex,

core/base-service/got.js

@@ -0,0 +1,96 @@
+import got from 'got'
+import { Inaccessible, InvalidResponse } from './errors.js'
+
+const userAgent = 'Shields.io/2003a'
+
+function requestOptions2GotOptions(options) {
+  const requestOptions = Object.assign({}, options)
+  const gotOptions = {}
+  const interchangableOptions = ['body', 'form', 'headers', 'method', 'url']
+
+  interchangableOptions.forEach(function (opt) {
+    if (opt in requestOptions) {
+      gotOptions[opt] = requestOptions[opt]
+      delete requestOptions[opt]
+    }
+  })
+
+  if ('qs' in requestOptions) {
+    gotOptions.searchParams = requestOptions.qs
+    delete requestOptions.qs
+  }
+
+  if ('gzip' in requestOptions) {
+    gotOptions.decompress = requestOptions.gzip
+    delete requestOptions.gzip
+  }
+
+  if ('strictSSL' in requestOptions) {
+    gotOptions.https = {
+      rejectUnauthorized: requestOptions.strictSSL,
+    }
+    delete requestOptions.strictSSL
+  }
+
+  if ('auth' in requestOptions) {
+    gotOptions.username = requestOptions.auth.user
+    gotOptions.password = requestOptions.auth.pass
+    delete requestOptions.auth
+  }
+
+  if (Object.keys(requestOptions).length > 0) {
+    throw new Error(`Found unrecognised options ${Object.keys(requestOptions)}`)
+  }
+
+  return gotOptions
+}
+
+async function sendRequest(gotWrapper, url, options) {
+  const gotOptions = requestOptions2GotOptions(options)
+  gotOptions.throwHttpErrors = false
+  gotOptions.retry = 0
+  gotOptions.headers = gotOptions.headers || {}
+  gotOptions.headers['User-Agent'] = userAgent
+  try {
+    const resp = await gotWrapper(url, gotOptions)
+    return { res: resp, buffer: resp.body }
+  } catch (err) {
+    if (err instanceof got.CancelError) {
+      throw new InvalidResponse({
+        underlyingError: new Error('Maximum response size exceeded'),
+      })
+    }
+    throw new Inaccessible({ underlyingError: err })
+  }
+}
+
+function fetchFactory(fetchLimitBytes) {
+  const gotWithLimit = got.extend({
+    handlers: [
+      (options, next) => {
+        const promiseOrStream = next(options)
+        promiseOrStream.on('downloadProgress', progress => {
+          if (
+            progress.transferred > fetchLimitBytes &&
+            // just accept the file if we've already finished downloading
+            // the entire file before we went over the limit
+            progress.percent !== 1
+          ) {
+            /*
+            TODO: we should be able to pass cancel() a message
+            https://github.com/sindresorhus/got/blob/main/documentation/advanced-creation.md#examples
+            but by the time we catch it, err.message is just "Promise was canceled"
+            */
+            promiseOrStream.cancel('Maximum response size exceeded')
+          }
+        })
+
+        return promiseOrStream
+      },
+    ],
+  })
+
+  return sendRequest.bind(sendRequest, gotWithLimit)
+}
+
+export { requestOptions2GotOptions, fetchFactory }

core/base-service/got.spec.js

@@ -0,0 +1,102 @@
+import { expect } from 'chai'
+import nock from 'nock'
+import { requestOptions2GotOptions, fetchFactory } from './got.js'
+import { Inaccessible, InvalidResponse } from './errors.js'
+
+describe('requestOptions2GotOptions function', function () {
+  it('translates valid options', function () {
+    expect(
+      requestOptions2GotOptions({
+        body: 'body',
+        form: 'form',
+        headers: 'headers',
+        method: 'method',
+        url: 'url',
+        qs: 'qs',
+        gzip: 'gzip',
+        strictSSL: 'strictSSL',
+        auth: { user: 'user', pass: 'pass' },
+      })
+    ).to.deep.equal({
+      body: 'body',
+      form: 'form',
+      headers: 'headers',
+      method: 'method',
+      url: 'url',
+      searchParams: 'qs',
+      decompress: 'gzip',
+      https: { rejectUnauthorized: 'strictSSL' },
+      username: 'user',
+      password: 'pass',
+    })
+  })
+
+  it('throws if unrecognised options are found', function () {
+    expect(() =>
+      requestOptions2GotOptions({ body: 'body', foobar: 'foobar' })
+    ).to.throw(Error, 'Found unrecognised options foobar')
+  })
+})
+
+describe('got wrapper', function () {
+  it('should not throw an error if the response <= fetchLimitBytes', async function () {
+    nock('https://www.google.com')
+      .get('/foo/bar')
+      .once()
+      .reply(200, 'x'.repeat(100))
+    const sendRequest = fetchFactory(100)
+    const { res } = await sendRequest('https://www.google.com/foo/bar')
+    expect(res.statusCode).to.equal(200)
+  })
+
+  it('should throw an InvalidResponse error if the response is > fetchLimitBytes', async function () {
+    nock('https://www.google.com')
+      .get('/foo/bar')
+      .once()
+      .reply(200, 'x'.repeat(101))
+    const sendRequest = fetchFactory(100)
+    return expect(
+      sendRequest('https://www.google.com/foo/bar')
+    ).to.be.rejectedWith(InvalidResponse, 'Maximum response size exceeded')
+  })
+
+  it('should throw an Inaccessible error if the request throws a (non-HTTP) error', async function () {
+    nock('https://www.google.com').get('/foo/bar').replyWithError('oh no')
+    const sendRequest = fetchFactory(1024)
+    return expect(
+      sendRequest('https://www.google.com/foo/bar')
+    ).to.be.rejectedWith(Inaccessible, 'oh no')
+  })
+
+  it('should throw an Inaccessible error if the host can not be accessed', async function () {
+    this.timeout(5000)
+    nock.disableNetConnect()
+    const sendRequest = fetchFactory(1024)
+    return expect(
+      sendRequest('https://www.google.com/foo/bar')
+    ).to.be.rejectedWith(
+      Inaccessible,
+      'Nock: Disallowed net connect for "www.google.com:443/foo/bar"'
+    )
+  })
+
+  it('should pass a custom user agent header', async function () {
+    nock('https://www.google.com', {
+      reqheaders: {
+        'user-agent': function (agent) {
+          return agent.startsWith('Shields.io')
+        },
+      },
+    })
+      .get('/foo/bar')
+      .once()
+      .reply(200)
+    const sendRequest = fetchFactory(1024)
+    await sendRequest('https://www.google.com/foo/bar')
+  })
+
+  afterEach(function () {
+    nock.cleanAll()
+    nock.enableNetConnect()
+  })
+})

core/base-service/legacy-result-sender.js

@@ -11,12 +11,14 @@ function streamFromString(str) {
 
 function sendSVG(res, askres, end) {
   askres.setHeader('Content-Type', 'image/svg+xml;charset=utf-8')
+  askres.setHeader('Content-Length', Buffer.byteLength(res, 'utf8'))
   end(null, { template: streamFromString(res) })
 }
 
 function sendJSON(res, askres, end) {
   askres.setHeader('Content-Type', 'application/json')
   askres.setHeader('Access-Control-Allow-Origin', '*')
+  askres.setHeader('Content-Length', Buffer.byteLength(res, 'utf8'))
   end(null, { template: streamFromString(res) })
 }