From 661da2a0e4911d011753038f345e3f07654b688f Mon Sep 17 00:00:00 2001 From: Phil Nash Date: Mon, 18 May 2020 13:06:27 +1000 Subject: [PATCH 1/7] refactor(client): uses TwilioServerlessApiClient instead of GotClient This will allow us to control the concurrency of requests made through the client at a later date. --- src/api/assets.ts | 29 ++++---- src/api/builds.ts | 64 ++++++++-------- src/api/environments.ts | 47 +++++++----- src/api/functions.ts | 39 +++++----- src/api/logs.ts | 20 ++--- src/api/services.ts | 21 +++--- src/api/utils/__tests__/pagination.test.ts | 20 ++--- src/api/utils/pagination.ts | 7 +- src/api/variables.ts | 24 +++--- src/client.ts | 86 +++++++++++++--------- src/streams/logs.ts | 11 +-- tsconfig.json | 3 +- 12 files changed, 202 insertions(+), 169 deletions(-) diff --git a/src/api/assets.ts b/src/api/assets.ts index cd6955d..723735e 100644 --- a/src/api/assets.ts +++ b/src/api/assets.ts @@ -1,18 +1,16 @@ /** @module @twilio-labs/serverless-api/dist/api */ -const { promisfy } = require('util'); - import debug from 'debug'; import FormData from 'form-data'; import { AssetApiResource, AssetList, AssetResource, - GotClient, ServerlessResourceConfig, Sid, VersionResource, } from '../types'; +import { TwilioServerlessApiClient } from '../client'; import { getContentType } from '../utils/content-type'; import { ClientApiError } from '../utils/error'; import { getApiUrl } from './utils/api-client'; @@ -25,16 +23,16 @@ const log = debug('twilio-serverless-api:assets'); * * @param {string} name friendly name of the resource * @param {string} serviceSid service to register asset under - * @param {GotClient} client API client + * @param {TwilioServerlessApiClient} client API client * @returns {Promise} */ async function createAssetResource( name: string, serviceSid: string, - client: GotClient + client: TwilioServerlessApiClient ): Promise { try { - const resp = await client.post(`Services/${serviceSid}/Assets`, { + const resp = await client.request('post', `Services/${serviceSid}/Assets`, { form: { FriendlyName: name, }, @@ -50,12 +48,12 @@ async function createAssetResource( * Calls the API to retrieve a list of all assets * * @param {string} serviceSid service to look for assets - * @param {GotClient} client API client + * @param {TwilioServerlessApiClient} client API client * @returns {Promise} */ export async function listAssetResources( serviceSid: string, - client: GotClient + client: TwilioServerlessApiClient ) { try { return getPaginatedResource( @@ -73,13 +71,13 @@ export async function listAssetResources( * * @param {FileInfo[]} assets * @param {string} serviceSid - * @param {GotClient} client + * @param {TwilioServerlessApiClient} client * @returns {Promise} */ export async function getOrCreateAssetResources( assets: ServerlessResourceConfig[], serviceSid: string, - client: GotClient + client: TwilioServerlessApiClient ): Promise { const output: AssetResource[] = []; const existingAssets = await listAssetResources(serviceSid, client); @@ -121,13 +119,13 @@ export async function getOrCreateAssetResources( * * @param {AssetResource} asset the one to create a new version for * @param {string} serviceSid the service to create the asset version for - * @param {GotClient} client API client + * @param {TwilioServerlessApiClient} client API client * @returns {Promise} */ async function createAssetVersion( asset: AssetResource, serviceSid: string, - client: GotClient + client: TwilioServerlessApiClient ): Promise { try { const contentType = await getContentType( @@ -146,7 +144,8 @@ async function createAssetVersion( form.append('Visibility', asset.access); form.append('Content', asset.content, contentOpts); - const resp = await client.post( + const resp = await client.requestText( + 'post', `Services/${serviceSid}/Assets/${asset.sid}/Versions`, { responseType: 'text', @@ -168,13 +167,13 @@ async function createAssetVersion( * @export * @param {AssetResource} asset The asset to upload * @param {string} serviceSid The service to upload it to - * @param {GotClient} client The API client + * @param {TwilioServerlessApiClient} client The API client * @returns {Promise} */ export async function uploadAsset( asset: AssetResource, serviceSid: string, - client: GotClient + client: TwilioServerlessApiClient ): Promise { const version = await createAssetVersion(asset, serviceSid, client); return version.sid; diff --git a/src/api/builds.ts b/src/api/builds.ts index 6d19f35..0974426 100644 --- a/src/api/builds.ts +++ b/src/api/builds.ts @@ -2,13 +2,8 @@ import debug from 'debug'; import querystring, { ParsedUrlQueryInput } from 'querystring'; -import { - BuildConfig, - BuildList, - BuildResource, - BuildStatus, - GotClient, -} from '../types'; +import { BuildConfig, BuildList, BuildResource, BuildStatus } from '../types'; +import { TwilioServerlessApiClient } from '../client'; import { DeployStatus } from '../types/consts'; import { ClientApiError } from '../utils/error'; import { sleep } from '../utils/sleep'; @@ -24,15 +19,18 @@ const log = debug('twilio-serverless-api:builds'); * @export * @param {string} buildSid SID of build to retrieve * @param {string} serviceSid service to retrieve build from - * @param {GotClient} client API client + * @param {TwilioServerlessApiClient} client API client * @returns {Promise} */ export async function getBuild( buildSid: string, serviceSid: string, - client: GotClient + client: TwilioServerlessApiClient ): Promise { - const resp = await client.get(`Services/${serviceSid}/Builds/${buildSid}`); + const resp = await client.request( + 'get', + `Services/${serviceSid}/Builds/${buildSid}` + ); return (resp.body as unknown) as BuildResource; } @@ -41,13 +39,13 @@ export async function getBuild( * * @param {string} buildSid the SID of the build * @param {string} serviceSid the SID of the service the build belongs to - * @param {GotClient} client API client + * @param {TwilioServerlessApiClient} client API client * @returns {Promise} */ async function getBuildStatus( buildSid: string, serviceSid: string, - client: GotClient + client: TwilioServerlessApiClient ): Promise { try { const resp = await getBuild(buildSid, serviceSid, client); @@ -63,12 +61,12 @@ async function getBuildStatus( * * @export * @param {string} serviceSid the SID of the service - * @param {GotClient} client API client + * @param {TwilioServerlessApiClient} client API client * @returns {Promise} */ export async function listBuilds( serviceSid: string, - client: GotClient + client: TwilioServerlessApiClient ): Promise { return getPaginatedResource( client, @@ -82,13 +80,13 @@ export async function listBuilds( * @export * @param {BuildConfig} config build-related information (functions, assets, dependencies) * @param {string} serviceSid the service to create the build for - * @param {GotClient} client API client + * @param {TwilioServerlessApiClient} client API client * @returns {Promise} */ export async function triggerBuild( config: BuildConfig, serviceSid: string, - client: GotClient + client: TwilioServerlessApiClient ): Promise { const { functionVersions, dependencies, assetVersions } = config; try { @@ -107,7 +105,7 @@ export async function triggerBuild( body.AssetVersions = assetVersions; } - const resp = await client.post(`Services/${serviceSid}/Builds`, { + const resp = await client.request('post', `Services/${serviceSid}/Builds`, { responseType: 'json', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -127,7 +125,7 @@ export async function triggerBuild( * @export * @param {string} buildSid the build to wait for * @param {string} serviceSid the service of the build - * @param {GotClient} client API client + * @param {TwilioServerlessApiClient} client API client * @param {events.EventEmitter} eventEmitter optional event emitter to communicate current build status * @param {number} timeout optional timeout. default: 5 minutes * @returns {Promise} @@ -135,8 +133,7 @@ export async function triggerBuild( export function waitForSuccessfulBuild( buildSid: string, serviceSid: string, - client: GotClient, - eventEmitter: events.EventEmitter, + client: TwilioServerlessApiClient, timeout: number = 5 * 60 * 1000 ): Promise { return new Promise(async (resolve, reject) => { @@ -145,12 +142,10 @@ export function waitForSuccessfulBuild( while (!isBuilt) { if (Date.now() - startTime > timeout) { - if (eventEmitter) { - eventEmitter.emit('status-update', { - status: DeployStatus.TIMED_OUT, - message: 'Deployment took too long', - }); - } + client.emit('status-update', { + status: DeployStatus.TIMED_OUT, + message: 'Deployment took too long', + }); reject(new Error('Timeout')); } const status = await getBuildStatus(buildSid, serviceSid, client); @@ -165,12 +160,10 @@ export function waitForSuccessfulBuild( return; } - if (eventEmitter) { - eventEmitter.emit('status-update', { - status: DeployStatus.BUILDING, - message: `Waiting for deployment. Current status: ${status}`, - }); - } + client.emit('status-update', { + status: DeployStatus.BUILDING, + message: `Waiting for deployment. Current status: ${status}`, + }); await sleep(1000); } resolve(); @@ -184,17 +177,18 @@ export function waitForSuccessfulBuild( * @param {string} buildSid the build to be activated * @param {string} environmentSid the target environment for the build to be deployed to * @param {string} serviceSid the service of the project - * @param {GotClient} client API client + * @param {TwilioServerlessApiClient} client API client * @returns {Promise} */ export async function activateBuild( buildSid: string, environmentSid: string, serviceSid: string, - client: GotClient + client: TwilioServerlessApiClient ): Promise { try { - const resp = await client.post( + const resp = await client.request( + 'post', `Services/${serviceSid}/Environments/${environmentSid}/Deployments`, { form: { diff --git a/src/api/environments.ts b/src/api/environments.ts index 893d932..a627d4d 100644 --- a/src/api/environments.ts +++ b/src/api/environments.ts @@ -4,6 +4,7 @@ import debug from 'debug'; import { EnvironmentList, EnvironmentResource, GotClient, Sid } from '../types'; import { getPaginatedResource } from './utils/pagination'; import { ClientApiError } from '../utils/error'; +import { TwilioServerlessApiClient } from '../client'; const log = debug('twilio-serverless-api:environments'); @@ -34,15 +35,16 @@ export function isEnvironmentSid(str: string) { * @export * @param {Sid} environmentSid the environment to retrieve * @param {Sid} serviceSid the service the environment belongs to - * @param {GotClient} client API client + * @param {TwilioServerlessApiClient} client API client * @returns {Promise} */ export async function getEnvironment( environmentSid: Sid, serviceSid: Sid, - client: GotClient + client: TwilioServerlessApiClient ): Promise { - const resp = await client.get( + const resp = await client.request( + 'get', `Services/${serviceSid}/Environments/${environmentSid}` ); return (resp.body as unknown) as EnvironmentResource; @@ -54,23 +56,27 @@ export async function getEnvironment( * @export * @param {string} domainSuffix the domain suffix for the environment * @param {string} serviceSid the service to create the environment for - * @param {GotClient} client API client + * @param {TwilioServerlessApiClient} client API client * @returns {Promise} */ export async function createEnvironmentFromSuffix( domainSuffix: string, serviceSid: string, - client: GotClient + client: TwilioServerlessApiClient ): Promise { const uniqueName = getUniqueNameFromSuffix(domainSuffix); - const resp = await client.post(`Services/${serviceSid}/Environments`, { - form: { - UniqueName: uniqueName, - DomainSuffix: domainSuffix || undefined, - // this property currently doesn't exist but for the future lets set it - FriendlyName: `${uniqueName} Environment (Created by CLI)`, - }, - }); + const resp = await client.request( + 'post', + `Services/${serviceSid}/Environments`, + { + form: { + UniqueName: uniqueName, + DomainSuffix: domainSuffix || undefined, + // this property currently doesn't exist but for the future lets set it + FriendlyName: `${uniqueName} Environment (Created by CLI)`, + }, + } + ); return (resp.body as unknown) as EnvironmentResource; } @@ -79,10 +85,13 @@ export async function createEnvironmentFromSuffix( * * @export * @param {string} serviceSid the service that the environments belong to - * @param {GotClient} client API client + * @param {TwilioServerlessApiClient} client API client * @returns */ -export async function listEnvironments(serviceSid: string, client: GotClient) { +export async function listEnvironments( + serviceSid: string, + client: TwilioServerlessApiClient +) { return getPaginatedResource( client, `Services/${serviceSid}/Environments` @@ -95,13 +104,13 @@ export async function listEnvironments(serviceSid: string, client: GotClient) { * @export * @param {string} domainSuffix the suffix to look for * @param {string} serviceSid the service the environment belongs to - * @param {GotClient} client API client + * @param {TwilioServerlessApiClient} client API client * @returns {Promise} */ export async function getEnvironmentFromSuffix( domainSuffix: string, serviceSid: string, - client: GotClient + client: TwilioServerlessApiClient ): Promise { const environments = await listEnvironments(serviceSid, client); let foundEnvironments = environments.filter( @@ -135,13 +144,13 @@ export async function getEnvironmentFromSuffix( * @export * @param {string} domainSuffix the domain suffix of the environment * @param {string} serviceSid the service the environment belongs to - * @param {GotClient} client API client + * @param {TwilioServerlessApiClient} client API client * @returns */ export async function createEnvironmentIfNotExists( domainSuffix: string, serviceSid: string, - client: GotClient + client: TwilioServerlessApiClient ) { return getEnvironmentFromSuffix(domainSuffix, serviceSid, client).catch( (err) => { diff --git a/src/api/functions.ts b/src/api/functions.ts index c54bd7a..0769d44 100644 --- a/src/api/functions.ts +++ b/src/api/functions.ts @@ -6,11 +6,11 @@ import { FunctionApiResource, FunctionList, FunctionResource, - GotClient, ServerlessResourceConfig, Sid, VersionResource, } from '../types'; +import { TwilioServerlessApiClient } from '../client'; import { getContentType } from '../utils/content-type'; import { ClientApiError } from '../utils/error'; import { getApiUrl } from './utils/api-client'; @@ -23,20 +23,24 @@ const log = debug('twilio-serverless-api:functions'); * * @param {string} name the friendly name of the function to create * @param {string} serviceSid the service the function should belong to - * @param {GotClient} client API client + * @param {TwilioServerlessApiClient} client API client * @returns {Promise} */ export async function createFunctionResource( name: string, serviceSid: string, - client: GotClient + client: TwilioServerlessApiClient ): Promise { try { - const resp = await client.post(`Services/${serviceSid}/Functions`, { - form: { - FriendlyName: name, - }, - }); + const resp = await client.request( + 'post', + `Services/${serviceSid}/Functions`, + { + form: { + FriendlyName: name, + }, + } + ); return (resp.body as unknown) as FunctionApiResource; } catch (err) { log('%O', new ClientApiError(err)); @@ -49,12 +53,12 @@ export async function createFunctionResource( * * @export * @param {string} serviceSid the service to look up - * @param {GotClient} client API client + * @param {TwilioServerlessApiClient} client API client * @returns */ export async function listFunctionResources( serviceSid: string, - client: GotClient + client: TwilioServerlessApiClient ) { try { return getPaginatedResource( @@ -73,13 +77,13 @@ export async function listFunctionResources( * @export * @param {FileInfo[]} functions list of functions to get or create * @param {string} serviceSid service the functions belong to - * @param {GotClient} client API client + * @param {TwilioServerlessApiClient} client API client * @returns {Promise} */ export async function getOrCreateFunctionResources( functions: ServerlessResourceConfig[], serviceSid: string, - client: GotClient + client: TwilioServerlessApiClient ): Promise { const output: FunctionResource[] = []; const existingFunctions = await listFunctionResources(serviceSid, client); @@ -121,13 +125,13 @@ export async function getOrCreateFunctionResources( * * @param {FunctionResource} fn the function the version should be created for * @param {string} serviceSid the service related to the function - * @param {GotClient} client API client + * @param {TwilioServerlessApiClient} client API client * @returns {Promise} */ async function createFunctionVersion( fn: FunctionResource, serviceSid: string, - client: GotClient + client: TwilioServerlessApiClient ): Promise { try { const contentType = @@ -145,7 +149,8 @@ async function createFunctionVersion( form.append('Visibility', fn.access); form.append('Content', fn.content, contentOpts); - const resp = await client.post( + const resp = await client.requestText( + 'post', `Services/${serviceSid}/Functions/${fn.sid}/Versions`, { responseType: 'text', @@ -167,13 +172,13 @@ async function createFunctionVersion( * @export * @param {FunctionResource} fn function to be uploaded * @param {string} serviceSid service that the function is connected to - * @param {GotClient} client API client + * @param {TwilioServerlessApiClient} client API client * @returns {Promise} */ export async function uploadFunction( fn: FunctionResource, serviceSid: string, - client: GotClient + client: TwilioServerlessApiClient ): Promise { const version = await createFunctionVersion(fn, serviceSid, client); return version.sid; diff --git a/src/api/logs.ts b/src/api/logs.ts index 67560e6..647474e 100644 --- a/src/api/logs.ts +++ b/src/api/logs.ts @@ -1,7 +1,8 @@ /** @module @twilio-labs/serverless-api/dist/api */ import debug from 'debug'; -import { GotClient, LogApiResource, LogList, Sid, LogFilters } from '../types'; +import { LogApiResource, LogList, Sid, LogFilters } from '../types'; +import { TwilioServerlessApiClient } from '../client'; import { getPaginatedResource } from './utils/pagination'; import { ClientApiError } from '../utils/error'; @@ -12,13 +13,13 @@ const log = debug('twilio-serverless-api:logs'); * * @param {Sid} environmentSid environment in which to get logs * @param {Sid} serviceSid service to look for logs - * @param {GotClient} client API client + * @param {TwilioServerlessApiClient} client API client * @returns {Promise} */ export async function listLogResources( environmentSid: Sid, serviceSid: Sid, - client: GotClient + client: TwilioServerlessApiClient ) { try { return getPaginatedResource( @@ -36,13 +37,13 @@ export async function listLogResources( * * @param {Sid} environmentSid environment in which to get logs * @param {Sid} serviceSid service to look for logs - * @param {GotClient} client API client + * @param {TwilioServerlessApiClient} client API client * @returns {Promise} */ export async function listOnePageLogResources( environmentSid: Sid, serviceSid: Sid, - client: GotClient, + client: TwilioServerlessApiClient, filters: LogFilters ): Promise { const pageSize = filters.pageSize || 50; @@ -65,7 +66,7 @@ export async function listOnePageLogResources( if (typeof pageToken !== 'undefined') { url += `&PageToken=${pageToken}`; } - const resp = await client.get(url); + const resp = await client.request('get', url); const content = (resp.body as unknown) as LogList; return content.logs as LogApiResource[]; } catch (err) { @@ -80,17 +81,18 @@ export async function listOnePageLogResources( * @param {Sid} logSid SID of log to retrieve * @param {Sid} environmentSid environment in which to get logs * @param {Sid} serviceSid service to look for logs - * @param {GotClient} client API client + * @param {TwilioServerlessApiClient} client API client * @returns {Promise} */ export async function getLog( logSid: Sid, environmentSid: Sid, serviceSid: Sid, - client: GotClient + client: TwilioServerlessApiClient ) { try { - const resp = await client.get( + const resp = await client.request( + 'get', `Services/${serviceSid}/Environments/${environmentSid}/Logs/${logSid}` ); return (resp.body as unknown) as LogApiResource; diff --git a/src/api/services.ts b/src/api/services.ts index 9905902..7999eba 100644 --- a/src/api/services.ts +++ b/src/api/services.ts @@ -1,7 +1,8 @@ /** @module @twilio-labs/serverless-api/dist/api */ import debug from 'debug'; -import { GotClient, ServiceList, ServiceResource, Sid } from '../types'; +import { ServiceList, ServiceResource, Sid } from '../types'; +import { TwilioServerlessApiClient } from '../client'; import { getPaginatedResource } from './utils/pagination'; import { ClientApiError } from '../utils/error'; @@ -12,15 +13,15 @@ const log = debug('twilio-serverless-api:services'); * * @export * @param {string} serviceName the unique name for the service - * @param {GotClient} client API client + * @param {TwilioServerlessApiClient} client API client * @returns {Promise} */ export async function createService( serviceName: string, - client: GotClient + client: TwilioServerlessApiClient ): Promise { try { - const resp = await client.post('Services', { + const resp = await client.request('post', 'Services', { form: { UniqueName: serviceName, FriendlyName: serviceName, @@ -40,11 +41,11 @@ export async function createService( * Lists all services attached to an account * * @export - * @param {GotClient} client API client + * @param {TwilioServerlessApiClient} client API client * @returns {Promise} */ export async function listServices( - client: GotClient + client: TwilioServerlessApiClient ): Promise { return getPaginatedResource(client, 'Services'); } @@ -54,12 +55,12 @@ export async function listServices( * * @export * @param {string} uniqueName the unique name of the service - * @param {GotClient} client API client + * @param {TwilioServerlessApiClient} client API client * @returns {(Promise)} */ export async function findServiceSid( uniqueName: string, - client: GotClient + client: TwilioServerlessApiClient ): Promise { try { const services = await listServices(client); @@ -77,10 +78,10 @@ export async function findServiceSid( export async function getService( sid: Sid, - client: GotClient + client: TwilioServerlessApiClient ): Promise { try { - const resp = await client.get(`Services/${sid}`); + const resp = await client.request('get', `Services/${sid}`); return (resp.body as unknown) as ServiceResource; } catch (err) { log('%O', new ClientApiError(err)); diff --git a/src/api/utils/__tests__/pagination.test.ts b/src/api/utils/__tests__/pagination.test.ts index 2133f22..9bbc36e 100644 --- a/src/api/utils/__tests__/pagination.test.ts +++ b/src/api/utils/__tests__/pagination.test.ts @@ -1,8 +1,8 @@ -import { createGotClient } from '../../../client'; +import { createGotClient, TwilioServerlessApiClient } from '../../../client'; import { ServiceList, ServiceResource } from '../../../types'; import { getPaginatedResource } from '../pagination'; -const client = createGotClient({ +const client = new TwilioServerlessApiClient({ accountSid: '', authToken: '', }); @@ -46,11 +46,11 @@ const baseResult: ServiceList = { describe('pagination', () => { beforeEach(() => { - client.get = jest.fn(); + client.request = jest.fn(); }); test('should return the results', async () => { - client.get = jest + client.request = jest .fn() .mockReturnValue(Promise.resolve({ body: baseResult })); @@ -59,7 +59,7 @@ describe('pagination', () => { '/Services' ); - expect(client.get).toHaveBeenCalledTimes(1); + expect(client.request).toHaveBeenCalledTimes(1); expect(results.length).toBe(3); expect(results[0].sid).toBe('ZS1'); expect(results[1].sid).toBe('ZS2'); @@ -71,7 +71,7 @@ describe('pagination', () => { responseBody.meta.next_page_url = 'https://next-page-url'; let pagesAvailable = 5; let idx = 0; - client.get = jest.fn().mockImplementation(() => { + client.request = jest.fn().mockImplementation(() => { const resp = { body: { ...baseResult } }; if (pagesAvailable > 1) { resp.body.meta.next_page_url = 'https://next-page-url'; @@ -91,7 +91,7 @@ describe('pagination', () => { '/Services' ); - expect(client.get).toHaveBeenCalledTimes(5); + expect(client.request).toHaveBeenCalledTimes(5); expect(results.length).toBe(5); expect(results[0].sid).toBe('ZS0'); expect(results[1].sid).toBe('ZS1'); @@ -102,7 +102,7 @@ describe('pagination', () => { test('should forward error on first try', async () => { const err = new Error('Test Error'); - client.get = jest.fn().mockImplementation(() => { + client.request = jest.fn().mockImplementation(() => { throw err; }); @@ -119,7 +119,7 @@ describe('pagination', () => { test('should ignore error on consec. requests', async () => { const err = new Error('Test Error'); let shouldThrow = false; - client.get = jest.fn().mockImplementation(() => { + client.request = jest.fn().mockImplementation(() => { if (shouldThrow) { throw err; } else { @@ -135,7 +135,7 @@ describe('pagination', () => { '/Services' ); - expect(client.get).toHaveBeenCalledTimes(2); + expect(client.request).toHaveBeenCalledTimes(2); expect(results.length).toBe(3); expect(results[0].sid).toBe('ZS1'); expect(results[1].sid).toBe('ZS2'); diff --git a/src/api/utils/pagination.ts b/src/api/utils/pagination.ts index 737a18b..77218ac 100644 --- a/src/api/utils/pagination.ts +++ b/src/api/utils/pagination.ts @@ -1,6 +1,7 @@ import debug from 'debug'; import { OptionsOfJSONResponseBody } from 'got'; -import { BaseList, GotClient } from '../../types'; +import { TwilioServerlessApiClient } from '../../client'; +import { BaseList } from '../../types'; import { ClientApiError } from '../../utils/error'; const log = debug('twilio-serverless-api:utils:pagination'); @@ -9,7 +10,7 @@ export async function getPaginatedResource< TList extends BaseList, TEntry >( - client: GotClient, + client: TwilioServerlessApiClient, url: string, opts: OptionsOfJSONResponseBody = { responseType: 'json' } ): Promise { @@ -25,7 +26,7 @@ export async function getPaginatedResource< if (nextPageUrl.startsWith('http')) { opts.prefixUrl = undefined; } - const resp = await client.get(nextPageUrl, opts); + const resp = await client.request('get', nextPageUrl, opts); const body = resp.body as TList; nextPageUrl = body.meta.next_page_url; const entries = body[body.meta.key] as TEntry[]; diff --git a/src/api/variables.ts b/src/api/variables.ts index 05fcea5..50e3439 100644 --- a/src/api/variables.ts +++ b/src/api/variables.ts @@ -3,11 +3,11 @@ import debug from 'debug'; import { EnvironmentVariables, - GotClient, Variable, VariableList, VariableResource, } from '../types'; +import { TwilioServerlessApiClient } from '../client'; import { getPaginatedResource } from './utils/pagination'; import { ClientApiError } from '../utils/error'; @@ -20,7 +20,7 @@ const log = debug('twilio-serverless-api:variables'); * @param {string} value the value of the variable * @param {string} environmentSid the environment the variable should be created for * @param {string} serviceSid the service that the environment belongs to - * @param {GotClient} client API client + * @param {TwilioServerlessApiClient} client API client * @returns {Promise} */ async function registerVariableInEnvironment( @@ -28,10 +28,11 @@ async function registerVariableInEnvironment( value: string, environmentSid: string, serviceSid: string, - client: GotClient + client: TwilioServerlessApiClient ): Promise { try { - const resp = await client.post( + const resp = await client.request( + 'post', `Services/${serviceSid}/Environments/${environmentSid}/Variables`, { form: { @@ -55,7 +56,7 @@ async function registerVariableInEnvironment( * @param {string} variableSid the SID of the existing variable * @param {string} environmentSid the environment the variable belongs to * @param {string} serviceSid the service the environment belongs to - * @param {GotClient} client API client + * @param {TwilioServerlessApiClient} client API client * @returns {Promise} */ async function updateVariableInEnvironment( @@ -64,10 +65,11 @@ async function updateVariableInEnvironment( variableSid: string, environmentSid: string, serviceSid: string, - client: GotClient + client: TwilioServerlessApiClient ): Promise { try { - const resp = await client.post( + const resp = await client.request( + 'post', `Services/${serviceSid}/Environments/${environmentSid}/Variables/${variableSid}`, { form: { @@ -89,13 +91,13 @@ async function updateVariableInEnvironment( * @export * @param {string} environmentSid the environment to get the variables for * @param {string} serviceSid the service the environment belongs to - * @param {GotClient} client API client + * @param {TwilioServerlessApiClient} client API client * @returns {Promise} */ export async function listVariablesForEnvironment( environmentSid: string, serviceSid: string, - client: GotClient + client: TwilioServerlessApiClient ): Promise { try { return getPaginatedResource( @@ -134,14 +136,14 @@ function convertToVariableArray(env: EnvironmentVariables): Variable[] { * @param {EnvironmentVariables} envVariables the object of variables * @param {string} environmentSid the environment the varibales should be set for * @param {string} serviceSid the service the environment belongs to - * @param {GotClient} client API client + * @param {TwilioServerlessApiClient} client API client * @returns {Promise} */ export async function setEnvironmentVariables( envVariables: EnvironmentVariables, environmentSid: string, serviceSid: string, - client: GotClient + client: TwilioServerlessApiClient ): Promise { const existingVariables = await listVariablesForEnvironment( environmentSid, diff --git a/src/client.ts b/src/client.ts index 9b7892d..d5ae26e 100644 --- a/src/client.ts +++ b/src/client.ts @@ -51,6 +51,11 @@ import { import { DeployStatus } from './types/consts'; import { ClientApiError, convertApiErrorsAndThrow } from './utils/error'; import { getListOfFunctionsAndAssets, SearchConfig } from './utils/fs'; +import { + HTTPAlias, + OptionsOfJSONResponseBody, + OptionsOfTextResponseBody, +} from 'got/dist/source'; const log = debug('twilio-serverless-api:client'); @@ -122,7 +127,7 @@ export class TwilioServerlessApiClient extends events.EventEmitter { types === 'services' || (types.length === 1 && types[0] === 'services') ) { - const services = await listServices(this.client); + const services = await listServices(this); return { services }; } @@ -130,7 +135,7 @@ export class TwilioServerlessApiClient extends events.EventEmitter { typeof serviceSid === 'undefined' && typeof serviceName !== 'undefined' ) { - serviceSid = await findServiceSid(serviceName, this.client); + serviceSid = await findServiceSid(serviceName, this); } if (typeof serviceSid === 'undefined') { @@ -144,14 +149,11 @@ export class TwilioServerlessApiClient extends events.EventEmitter { for (const type of types) { try { if (type === 'environments') { - result.environments = await listEnvironments( - serviceSid, - this.client - ); + result.environments = await listEnvironments(serviceSid, this); } if (type === 'builds') { - result.builds = await listBuilds(serviceSid, this.client); + result.builds = await listBuilds(serviceSid, this); } if (typeof environmentSid === 'string') { @@ -159,7 +161,7 @@ export class TwilioServerlessApiClient extends events.EventEmitter { const environment = await getEnvironmentFromSuffix( environmentSid, serviceSid, - this.client + this ); environmentSid = environment.sid; currentBuildSidForEnv = environment.build_sid; @@ -167,7 +169,7 @@ export class TwilioServerlessApiClient extends events.EventEmitter { const environment = await getEnvironment( environmentSid, serviceSid, - this.client + this ); currentBuildSidForEnv = environment.build_sid; } @@ -177,7 +179,7 @@ export class TwilioServerlessApiClient extends events.EventEmitter { currentBuild = await getBuild( currentBuildSidForEnv, serviceSid, - this.client + this ); } @@ -199,7 +201,7 @@ export class TwilioServerlessApiClient extends events.EventEmitter { entries: await listVariablesForEnvironment( environmentSid, serviceSid, - this.client + this ), environmentSid, }; @@ -223,7 +225,7 @@ export class TwilioServerlessApiClient extends events.EventEmitter { const environmentResource = await getEnvironmentFromSuffix( environment, serviceSid, - this.client + this ); environment = environmentResource.sid; } @@ -231,7 +233,7 @@ export class TwilioServerlessApiClient extends events.EventEmitter { if (filterByFunction && !isFunctionSid(filterByFunction)) { const availableFunctions = await listFunctionResources( serviceSid, - this.client + this ); const foundFunction = availableFunctions.find( (fn) => fn.friendly_name === filterByFunction @@ -244,7 +246,7 @@ export class TwilioServerlessApiClient extends events.EventEmitter { const logsStream = new LogsStream( environment, serviceSid, - this.client, + this, logsConfig ); @@ -261,7 +263,7 @@ export class TwilioServerlessApiClient extends events.EventEmitter { const environmentResource = await getEnvironmentFromSuffix( environment, serviceSid, - this.client + this ); environment = environmentResource.sid; } @@ -269,7 +271,7 @@ export class TwilioServerlessApiClient extends events.EventEmitter { if (filterByFunction && !isFunctionSid(filterByFunction)) { const availableFunctions = await listFunctionResources( serviceSid, - this.client + this ); const foundFunction = availableFunctions.find( (fn) => fn.friendly_name === filterByFunction @@ -280,7 +282,7 @@ export class TwilioServerlessApiClient extends events.EventEmitter { filterByFunction = foundFunction.sid; } - return listOnePageLogResources(environment, serviceSid, this.client, { + return listOnePageLogResources(environment, serviceSid, this, { pageSize: 50, functionSid: filterByFunction, }); @@ -319,7 +321,7 @@ export class TwilioServerlessApiClient extends events.EventEmitter { const environment = await getEnvironmentFromSuffix( targetEnvironment, serviceSid, - this.client + this ); targetEnvironment = environment.sid; } catch (err) { @@ -327,7 +329,7 @@ export class TwilioServerlessApiClient extends events.EventEmitter { const environment = await createEnvironmentFromSuffix( targetEnvironment, serviceSid, - this.client + this ); targetEnvironment = environment.sid; } else { @@ -342,13 +344,13 @@ export class TwilioServerlessApiClient extends events.EventEmitter { currentEnv = await getEnvironmentFromSuffix( sourceEnvironment, serviceSid, - this.client + this ); } else { currentEnv = await getEnvironment( sourceEnvironment, serviceSid, - this.client + this ); } buildSid = currentEnv.build_sid; @@ -361,9 +363,9 @@ export class TwilioServerlessApiClient extends events.EventEmitter { const { domain_name } = await getEnvironment( targetEnvironment, serviceSid, - this.client + this ); - await activateBuild(buildSid, targetEnvironment, serviceSid, this.client); + await activateBuild(buildSid, targetEnvironment, serviceSid, this); return { serviceSid, @@ -415,11 +417,11 @@ export class TwilioServerlessApiClient extends events.EventEmitter { message: 'Creating Service', }); try { - serviceSid = await createService(config.serviceName, this.client); + serviceSid = await createService(config.serviceName, this); } catch (err) { const alternativeServiceSid = await findServiceSid( config.serviceName, - this.client + this ); if (!alternativeServiceSid) { throw err; @@ -451,7 +453,7 @@ export class TwilioServerlessApiClient extends events.EventEmitter { const environment = await createEnvironmentIfNotExists( config.functionsEnv, serviceSid, - this.client + this ); const { sid: environmentSid, domain_name: domain } = environment; @@ -466,7 +468,7 @@ export class TwilioServerlessApiClient extends events.EventEmitter { const functionResources = await getOrCreateFunctionResources( functions, serviceSid, - this.client + this ); this.emit('status-update', { @@ -475,7 +477,7 @@ export class TwilioServerlessApiClient extends events.EventEmitter { }); const functionVersions = await Promise.all( functionResources.map((fn) => { - return uploadFunction(fn, serviceSid as string, this.client); + return uploadFunction(fn, serviceSid as string, this); }) ); @@ -490,7 +492,7 @@ export class TwilioServerlessApiClient extends events.EventEmitter { const assetResources = await getOrCreateAssetResources( assets, serviceSid, - this.client + this ); this.emit('status-update', { @@ -499,7 +501,7 @@ export class TwilioServerlessApiClient extends events.EventEmitter { }); const assetVersions = await Promise.all( assetResources.map((asset) => { - return uploadAsset(asset, serviceSid as string, this.client); + return uploadAsset(asset, serviceSid as string, this); }) ); @@ -511,9 +513,9 @@ export class TwilioServerlessApiClient extends events.EventEmitter { const build = await triggerBuild( { functionVersions, dependencies, assetVersions }, serviceSid, - this.client + this ); - await waitForSuccessfulBuild(build.sid, serviceSid, this.client, this); + await waitForSuccessfulBuild(build.sid, serviceSid, this); this.emit('status-update', { status: DeployStatus.SETTING_VARIABLES, @@ -523,14 +525,14 @@ export class TwilioServerlessApiClient extends events.EventEmitter { config.env, environmentSid, serviceSid, - this.client + this ); this.emit('status-update', { status: DeployStatus.ACTIVATING_DEPLOYMENT, message: 'Activating deployment', }); - await activateBuild(build.sid, environmentSid, serviceSid, this.client); + await activateBuild(build.sid, environmentSid, serviceSid, this); this.emit('status', { status: DeployStatus.DONE, @@ -607,6 +609,22 @@ export class TwilioServerlessApiClient extends events.EventEmitter { convertApiErrorsAndThrow(err); } } + + async request( + method: HTTPAlias, + path: string, + options?: OptionsOfJSONResponseBody + ) { + return this.client[method](path, options); + } + + async requestText( + method: HTTPAlias, + path: string, + options?: OptionsOfTextResponseBody + ) { + return this.client[method](path, options); + } } export default TwilioServerlessApiClient; diff --git a/src/streams/logs.ts b/src/streams/logs.ts index 93d106e..360e6e6 100644 --- a/src/streams/logs.ts +++ b/src/streams/logs.ts @@ -1,6 +1,7 @@ import { Readable } from 'stream'; import { listOnePageLogResources } from '../api/logs'; -import { LogApiResource, Sid, GotClient } from '../types'; +import { Sid } from '../types'; +import { TwilioServerlessApiClient } from '../client'; import { LogsConfig } from '../types/logs'; export class LogsStream extends Readable { @@ -11,7 +12,7 @@ export class LogsStream extends Readable { constructor( private environmentSid: Sid, private serviceSid: Sid, - private client: GotClient, + private client: TwilioServerlessApiClient, private config: LogsConfig ) { super({ objectMode: true }); @@ -42,9 +43,9 @@ export class LogsStream extends Readable { } ); logs - .filter(log => !this._viewedSids.has(log.sid)) + .filter((log) => !this._viewedSids.has(log.sid)) .reverse() - .forEach(log => { + .forEach((log) => { this.push(log); }); // Replace the set each time rather than adding to the set. @@ -52,7 +53,7 @@ export class LogsStream extends Readable { // will either overlap or not. This is instead of keeping an ever growing // set of viewSids which would cause memory issues for long running log // tails. - this._viewedSids = new Set(logs.map(log => log.sid)); + this._viewedSids = new Set(logs.map((log) => log.sid)); if (!this.config.tail) { this.push(null); } diff --git a/tsconfig.json b/tsconfig.json index 203329c..8b3a3d1 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -46,7 +46,7 @@ // "typeRoots": [], /* List of folders to include type definitions from. */ // "types": [], /* Type declaration files to be included in compilation. */ // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ - "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */ + "esModuleInterop": true, /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */ // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ /* Source Map Options */ @@ -58,5 +58,6 @@ /* Experimental Options */ // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ + "skipLibCheck": true } } From 5be2d10accd035aa6766a4883f6287b728b499b3 Mon Sep 17 00:00:00 2001 From: Phil Nash Date: Tue, 19 May 2020 10:57:06 +1000 Subject: [PATCH 2/7] feat(client): adds p-limit to limit concurrency Has a default concurrency of 50, but is configurable in the TwilioServerlessApiClient constructor. --- package.json | 1 + src/client.ts | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 1b2379c..56cd544 100644 --- a/package.json +++ b/package.json @@ -70,6 +70,7 @@ "form-data": "^2.5.0", "got": "^11.0.1", "mime-types": "^2.1.22", + "p-limit": "^2.3.0", "recursive-readdir": "^2.2.2", "type-fest": "^0.3.0", "upath": "^1.1.2" diff --git a/src/client.ts b/src/client.ts index d5ae26e..3cf1711 100644 --- a/src/client.ts +++ b/src/client.ts @@ -56,6 +56,7 @@ import { OptionsOfJSONResponseBody, OptionsOfTextResponseBody, } from 'got/dist/source'; +import pLimit, { Limit } from 'p-limit'; const log = debug('twilio-serverless-api:client'); @@ -92,11 +93,20 @@ export class TwilioServerlessApiClient extends events.EventEmitter { */ private client: GotClient; - constructor(config: ClientConfig) { + /** + * + * @private + * @type {Limit} + * @memberof TwilioServerlessApiClient + */ + private limit: Limit; + + constructor(config: ClientConfig, concurrency = 50) { debug.enable(process.env.DEBUG || ''); super(); this.config = config; this.client = createGotClient(config); + this.limit = pLimit(concurrency); } /** @@ -615,7 +625,7 @@ export class TwilioServerlessApiClient extends events.EventEmitter { path: string, options?: OptionsOfJSONResponseBody ) { - return this.client[method](path, options); + return this.limit(() => this.client[method](path, options)); } async requestText( @@ -623,7 +633,7 @@ export class TwilioServerlessApiClient extends events.EventEmitter { path: string, options?: OptionsOfTextResponseBody ) { - return this.client[method](path, options); + return this.limit(() => this.client[method](path, options)); } } From d7300bf73c9146fb1775a0bc9e9254383d260ae6 Mon Sep 17 00:00:00 2001 From: Phil Nash Date: Tue, 19 May 2020 12:11:56 +1000 Subject: [PATCH 3/7] feat(client): increases retry limit for failed HTTP requests Got already has retry capabilities, with back off, built in. Default number of retries is 2, so this increases to 10. It also only retries on certain HTTP statuses (including 429) and other network errors. --- src/client.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/client.ts b/src/client.ts index 3cf1711..4339ff7 100644 --- a/src/client.ts +++ b/src/client.ts @@ -58,6 +58,10 @@ import { } from 'got/dist/source'; import pLimit, { Limit } from 'p-limit'; +const RETRY_DEFAULTS = { + limit: 10, +}; + const log = debug('twilio-serverless-api:client'); export function createGotClient(config: ClientConfig): GotClient { @@ -623,16 +627,18 @@ export class TwilioServerlessApiClient extends events.EventEmitter { async request( method: HTTPAlias, path: string, - options?: OptionsOfJSONResponseBody + options: OptionsOfJSONResponseBody = {} ) { + options.retry = RETRY_DEFAULTS; return this.limit(() => this.client[method](path, options)); } async requestText( method: HTTPAlias, path: string, - options?: OptionsOfTextResponseBody + options: OptionsOfTextResponseBody = {} ) { + options.retry = RETRY_DEFAULTS; return this.limit(() => this.client[method](path, options)); } } From a53fdb63671078b9690b36c698a90bcacff4422d Mon Sep 17 00:00:00 2001 From: Phil Nash Date: Tue, 19 May 2020 12:41:11 +1000 Subject: [PATCH 4/7] fix(client): no need to attach config to got client Since we no longer pass the GotClient around, adding the ClientConfig to it doesn't help. Instead, we pass the config to functions that need it. --- src/__tests__/client.test.ts | 3 --- src/api/assets.ts | 16 ++++++++++++---- src/api/functions.ts | 16 ++++++++++++---- src/client.ts | 5 ++--- src/types/generic.ts | 4 +--- 5 files changed, 27 insertions(+), 17 deletions(-) diff --git a/src/__tests__/client.test.ts b/src/__tests__/client.test.ts index 06a845a..79aa17d 100644 --- a/src/__tests__/client.test.ts +++ b/src/__tests__/client.test.ts @@ -14,7 +14,6 @@ describe('createGotClient', () => { expect((options as any).password).toBe( DEFAULT_TEST_CLIENT_CONFIG.authToken ); - expect(client.twilioClientConfig).toEqual(config); }); test('works with region configuration', () => { @@ -32,7 +31,6 @@ describe('createGotClient', () => { expect((options as any).password).toBe( DEFAULT_TEST_CLIENT_CONFIG.authToken ); - expect(client.twilioClientConfig).toEqual(config); }); test('works with region & edge configuration', () => { @@ -53,6 +51,5 @@ describe('createGotClient', () => { expect((options as any).password).toBe( DEFAULT_TEST_CLIENT_CONFIG.authToken ); - expect(client.twilioClientConfig).toEqual(config); }); }); diff --git a/src/api/assets.ts b/src/api/assets.ts index 723735e..35a5ca0 100644 --- a/src/api/assets.ts +++ b/src/api/assets.ts @@ -9,6 +9,7 @@ import { ServerlessResourceConfig, Sid, VersionResource, + ClientConfig, } from '../types'; import { TwilioServerlessApiClient } from '../client'; import { getContentType } from '../utils/content-type'; @@ -125,7 +126,8 @@ export async function getOrCreateAssetResources( async function createAssetVersion( asset: AssetResource, serviceSid: string, - client: TwilioServerlessApiClient + client: TwilioServerlessApiClient, + clientConfig: ClientConfig ): Promise { try { const contentType = await getContentType( @@ -149,7 +151,7 @@ async function createAssetVersion( `Services/${serviceSid}/Assets/${asset.sid}/Versions`, { responseType: 'text', - prefixUrl: getApiUrl(client.twilioClientConfig, 'serverless-upload'), + prefixUrl: getApiUrl(clientConfig, 'serverless-upload'), body: form, } ); @@ -173,8 +175,14 @@ async function createAssetVersion( export async function uploadAsset( asset: AssetResource, serviceSid: string, - client: TwilioServerlessApiClient + client: TwilioServerlessApiClient, + clientConfig: ClientConfig ): Promise { - const version = await createAssetVersion(asset, serviceSid, client); + const version = await createAssetVersion( + asset, + serviceSid, + client, + clientConfig + ); return version.sid; } diff --git a/src/api/functions.ts b/src/api/functions.ts index 0769d44..0e64568 100644 --- a/src/api/functions.ts +++ b/src/api/functions.ts @@ -9,6 +9,7 @@ import { ServerlessResourceConfig, Sid, VersionResource, + ClientConfig, } from '../types'; import { TwilioServerlessApiClient } from '../client'; import { getContentType } from '../utils/content-type'; @@ -131,7 +132,8 @@ export async function getOrCreateFunctionResources( async function createFunctionVersion( fn: FunctionResource, serviceSid: string, - client: TwilioServerlessApiClient + client: TwilioServerlessApiClient, + clientConfig: ClientConfig ): Promise { try { const contentType = @@ -154,7 +156,7 @@ async function createFunctionVersion( `Services/${serviceSid}/Functions/${fn.sid}/Versions`, { responseType: 'text', - prefixUrl: getApiUrl(client.twilioClientConfig, 'serverless-upload'), + prefixUrl: getApiUrl(clientConfig, 'serverless-upload'), body: form, } ); @@ -178,9 +180,15 @@ async function createFunctionVersion( export async function uploadFunction( fn: FunctionResource, serviceSid: string, - client: TwilioServerlessApiClient + client: TwilioServerlessApiClient, + clientConfig: ClientConfig ): Promise { - const version = await createFunctionVersion(fn, serviceSid, client); + const version = await createFunctionVersion( + fn, + serviceSid, + client, + clientConfig + ); return version.sid; } diff --git a/src/client.ts b/src/client.ts index 4339ff7..c06c8e9 100644 --- a/src/client.ts +++ b/src/client.ts @@ -74,7 +74,6 @@ export function createGotClient(config: ClientConfig): GotClient { 'User-Agent': 'twilio-serverless-api', }, }) as GotClient; - client.twilioClientConfig = config; return client; } @@ -491,7 +490,7 @@ export class TwilioServerlessApiClient extends events.EventEmitter { }); const functionVersions = await Promise.all( functionResources.map((fn) => { - return uploadFunction(fn, serviceSid as string, this); + return uploadFunction(fn, serviceSid as string, this, this.config); }) ); @@ -515,7 +514,7 @@ export class TwilioServerlessApiClient extends events.EventEmitter { }); const assetVersions = await Promise.all( assetResources.map((asset) => { - return uploadAsset(asset, serviceSid as string, this); + return uploadAsset(asset, serviceSid as string, this, this.config); }) ); diff --git a/src/types/generic.ts b/src/types/generic.ts index 1b98f4f..2144d3e 100644 --- a/src/types/generic.ts +++ b/src/types/generic.ts @@ -3,9 +3,7 @@ import { Got } from 'got'; import { ClientConfig } from './client'; -export type GotClient = Got & { - twilioClientConfig: ClientConfig; -}; +export type GotClient = Got; export type EnvironmentVariables = { [key: string]: string | undefined; From 647afefb2d80e238c1a8aaa716223fec254ce816 Mon Sep 17 00:00:00 2001 From: Phil Nash Date: Thu, 21 May 2020 10:39:53 +1000 Subject: [PATCH 5/7] chore(builds): add eventEmitter back to waitForSuccessfulBuild --- src/api/builds.ts | 5 +++-- src/client.ts | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/api/builds.ts b/src/api/builds.ts index 0974426..e8b6578 100644 --- a/src/api/builds.ts +++ b/src/api/builds.ts @@ -134,6 +134,7 @@ export function waitForSuccessfulBuild( buildSid: string, serviceSid: string, client: TwilioServerlessApiClient, + eventEmitter: events.EventEmitter, timeout: number = 5 * 60 * 1000 ): Promise { return new Promise(async (resolve, reject) => { @@ -142,7 +143,7 @@ export function waitForSuccessfulBuild( while (!isBuilt) { if (Date.now() - startTime > timeout) { - client.emit('status-update', { + eventEmitter.emit('status-update', { status: DeployStatus.TIMED_OUT, message: 'Deployment took too long', }); @@ -160,7 +161,7 @@ export function waitForSuccessfulBuild( return; } - client.emit('status-update', { + eventEmitter.emit('status-update', { status: DeployStatus.BUILDING, message: `Waiting for deployment. Current status: ${status}`, }); diff --git a/src/client.ts b/src/client.ts index c06c8e9..2b4f7d9 100644 --- a/src/client.ts +++ b/src/client.ts @@ -528,7 +528,7 @@ export class TwilioServerlessApiClient extends events.EventEmitter { serviceSid, this ); - await waitForSuccessfulBuild(build.sid, serviceSid, this); + await waitForSuccessfulBuild(build.sid, serviceSid, this, this); this.emit('status-update', { status: DeployStatus.SETTING_VARIABLES, From 3d87d09c5756aa1d970bfa0a44234f2797b9f7a0 Mon Sep 17 00:00:00 2001 From: Phil Nash Date: Thu, 21 May 2020 11:46:25 +1000 Subject: [PATCH 6/7] feat(client): concurrency and retry options configurable Can configure in constructor or environment. --- README.md | 49 ++++++++++++++++++++++++++---------- src/api/utils/http_config.ts | 12 +++++++++ src/client.ts | 17 +++++++------ src/types/client.ts | 8 ++++++ 4 files changed, 65 insertions(+), 21 deletions(-) create mode 100644 src/api/utils/http_config.ts diff --git a/README.md b/README.md index 86c249a..87204d7 100644 --- a/README.md +++ b/README.md @@ -6,19 +6,20 @@ Part of the Serverle npm (scoped) npm GitHub All Contributors Code of Conduct PRs Welcome
-- [Installation](#installation) -- [Example](#example) -- [API](#api) - - [`client.activateBuild(activateConfig: ActivateConfig): Promise`](#clientactivatebuildactivateconfig-activateconfig-promiseactivateresult) - - [`client.deployLocalProject(deployConfig: DeployLocalProjectConfig): Promise`](#clientdeploylocalprojectdeployconfig-deploylocalprojectconfig-promisedeployresult) - - [`client.deployProject(deployConfig: DeployProjectConfig): Promise`](#clientdeployprojectdeployconfig-deployprojectconfig-promisedeployresult) - - [`client.getClient(): GotClient`](#clientgetclient-gotclient) - - [`client.list(listConfig: ListConfig): Promise`](#clientlistlistconfig-listconfig-promiselistresult) - - [`api` and `fsHelpers`](#api-and-fshelpers) -- [Contributing](#contributing) - - [Code of Conduct](#code-of-conduct) - - [Contributors](#contributors) -- [License](#license) +* [Installation](#installation) +* [Example](#example) +* [HTTP Client Configuration](#http-client-configuration) +* [API](#api) + * [`client.activateBuild(activateConfig: ActivateConfig): Promise`](#clientactivatebuildactivateconfig-activateconfig-promiseactivateresult) + * [`client.deployLocalProject(deployConfig: DeployLocalProjectConfig): Promise`](#clientdeploylocalprojectdeployconfig-deploylocalprojectconfig-promisedeployresult) + * [`client.deployProject(deployConfig: DeployProjectConfig): Promise`](#clientdeployprojectdeployconfig-deployprojectconfig-promisedeployresult) + * [`client.getClient(): GotClient`](#clientgetclient-gotclient) + * [`client.list(listConfig: ListConfig): Promise`](#clientlistlistconfig-listconfig-promiselistresult) + * [`api` and `fsHelpers`](#api-and-fshelpers) +* [Contributing](#contributing) + * [Code of Conduct](#code-of-conduct) + * [Contributors](#contributors) +* [License](#license) ## Installation @@ -56,6 +57,28 @@ const result = await client.deployLocalProject({ }); ``` +## HTTP Client Configuration + +When deploying lots of Functions and Assets it is possible to run up against the enforced concurrency limits of the Twilio API. You can limit the concurrency and set how many times the library retries API requests either in the constructor for `TwilioServerlessApiClient` or using environment variables (useful when this is part of a CLI tool like `twilio-run`). + +The default concurrency is 50 and the default number of retries is 10. You can change this in the config, the following would set concurrency to 1, only 1 live request at a time, and retries to 0, so if it fails it won't retry. + +```js +const client = new TwilioServerlessApiClient({ + accountSid: '...', + authToken: '...', + concurrency: 1, + retryLimit: 0 +};); +``` + +You can also set these values with the following environment variables: + +```bash +export TWILIO_SERVERLESS_API_CONCURRENCY=1 +export TWILIO_SERVERLESS_API_RETRY_LIMIT=0 +``` + ## API You can find the full reference documentation of everything at: https://serverless-api.twilio-labs.com diff --git a/src/api/utils/http_config.ts b/src/api/utils/http_config.ts new file mode 100644 index 0000000..f365a2d --- /dev/null +++ b/src/api/utils/http_config.ts @@ -0,0 +1,12 @@ +let retryLimit = process.env.TWILIO_SERVERLESS_API_RETRY_LIMIT; +if (typeof retryLimit === 'undefined') { + retryLimit = '10'; +} + +let concurrency = process.env.TWILIO_SERVERLESS_API_CONCURRENCY; +if (typeof concurrency === 'undefined') { + concurrency = '50'; +} + +export const RETRY_LIMIT = parseInt(retryLimit, 10); +export const CONCURRENCY = parseInt(concurrency, 10); diff --git a/src/client.ts b/src/client.ts index 2b4f7d9..e298882 100644 --- a/src/client.ts +++ b/src/client.ts @@ -28,6 +28,7 @@ import { import { listOnePageLogResources } from './api/logs'; import { createService, findServiceSid, listServices } from './api/services'; import { getApiUrl } from './api/utils/api-client'; +import { RETRY_LIMIT, CONCURRENCY } from './api/utils/http_config'; import { listVariablesForEnvironment, setEnvironmentVariables, @@ -58,10 +59,6 @@ import { } from 'got/dist/source'; import pLimit, { Limit } from 'p-limit'; -const RETRY_DEFAULTS = { - limit: 10, -}; - const log = debug('twilio-serverless-api:client'); export function createGotClient(config: ClientConfig): GotClient { @@ -104,12 +101,12 @@ export class TwilioServerlessApiClient extends events.EventEmitter { */ private limit: Limit; - constructor(config: ClientConfig, concurrency = 50) { + constructor(config: ClientConfig) { debug.enable(process.env.DEBUG || ''); super(); this.config = config; this.client = createGotClient(config); - this.limit = pLimit(concurrency); + this.limit = pLimit(config.concurrency || CONCURRENCY); } /** @@ -628,7 +625,9 @@ export class TwilioServerlessApiClient extends events.EventEmitter { path: string, options: OptionsOfJSONResponseBody = {} ) { - options.retry = RETRY_DEFAULTS; + options.retry = { + limit: this.config.retryLimit || RETRY_LIMIT, + }; return this.limit(() => this.client[method](path, options)); } @@ -637,7 +636,9 @@ export class TwilioServerlessApiClient extends events.EventEmitter { path: string, options: OptionsOfTextResponseBody = {} ) { - options.retry = RETRY_DEFAULTS; + options.retry = { + limit: this.config.retryLimit || RETRY_LIMIT, + }; return this.limit(() => this.client[method](path, options)); } } diff --git a/src/types/client.ts b/src/types/client.ts index f9564be..57d89a3 100644 --- a/src/types/client.ts +++ b/src/types/client.ts @@ -20,4 +20,12 @@ export type ClientConfig = { * Twilio Edge */ edge?: string; + /** + * Limit concurrency + */ + concurrency?: number; + /** + * Number of retry attempts the client will make on a failure + */ + retryLimit?: number; }; From 83bd9703eb6bf8b8770182ffa0c885fa7b18ce1f Mon Sep 17 00:00:00 2001 From: Phil Nash Date: Thu, 21 May 2020 12:00:11 +1000 Subject: [PATCH 7/7] feat(client): fixes types for request method --- src/api/assets.ts | 2 +- src/api/functions.ts | 2 +- src/client.ts | 28 +++++++++++++++++----------- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/api/assets.ts b/src/api/assets.ts index 35a5ca0..b3d88a1 100644 --- a/src/api/assets.ts +++ b/src/api/assets.ts @@ -146,7 +146,7 @@ async function createAssetVersion( form.append('Visibility', asset.access); form.append('Content', asset.content, contentOpts); - const resp = await client.requestText( + const resp = await client.request( 'post', `Services/${serviceSid}/Assets/${asset.sid}/Versions`, { diff --git a/src/api/functions.ts b/src/api/functions.ts index 0e64568..e018a76 100644 --- a/src/api/functions.ts +++ b/src/api/functions.ts @@ -151,7 +151,7 @@ async function createFunctionVersion( form.append('Visibility', fn.access); form.append('Content', fn.content, contentOpts); - const resp = await client.requestText( + const resp = await client.request( 'post', `Services/${serviceSid}/Functions/${fn.sid}/Versions`, { diff --git a/src/client.ts b/src/client.ts index e298882..1f73349 100644 --- a/src/client.ts +++ b/src/client.ts @@ -56,6 +56,8 @@ import { HTTPAlias, OptionsOfJSONResponseBody, OptionsOfTextResponseBody, + Options, + Response, } from 'got/dist/source'; import pLimit, { Limit } from 'p-limit'; @@ -620,22 +622,26 @@ export class TwilioServerlessApiClient extends events.EventEmitter { } } + // request json without options + async request(method: HTTPAlias, path: string): Promise>; + // request json with options async request( method: HTTPAlias, path: string, - options: OptionsOfJSONResponseBody = {} - ) { - options.retry = { - limit: this.config.retryLimit || RETRY_LIMIT, - }; - return this.limit(() => this.client[method](path, options)); - } - - async requestText( + options: OptionsOfJSONResponseBody + ): Promise>; + // request text + async request( + method: HTTPAlias, + path: string, + options: OptionsOfTextResponseBody + ): Promise>; + // general implementation + async request( method: HTTPAlias, path: string, - options: OptionsOfTextResponseBody = {} - ) { + options: Options = {} + ): Promise { options.retry = { limit: this.config.retryLimit || RETRY_LIMIT, };