feat(logs): add logs fetching and streaming

package.json

@@ -8,6 +8,7 @@
     "jest": "jest",
     "test": "run-s build:noemit jest",
     "build": "tsc",
+    "watch": "tsc --watch",
     "build:noemit": "tsc --noEmit",
     "postbuild": "npm shrinkwrap",
     "docs": "typedoc --options typedoc.json",
@@ -16,7 +17,7 @@
     "lint-staged": "lint-staged",
     "cm": "git-cz",
     "release": "HUSKY_SKIP_HOOKS=1 standard-version",
-    "reset-dependencies": "bash ./hooks/update-dependencies"
+    "reset-dependencies": "bash ./hooks/update-dependencies.sh"
   },
   "publishConfig": {
     "access": "public"

src/api/functions.ts

@@ -180,3 +180,14 @@ export async function uploadFunction(
   const version = await createFunctionVersion(fn, serviceSid, client);
   return version.sid;
 }
+
+/**
+ * Checks if a string is an function SID by checking its prefix and length
+ *
+ * @export
+ * @param {string} str the string to check
+ * @returns
+ */
+export function isFunctionSid(str: string) {
+  return str.startsWith('ZH') && str.length === 34;
+}

src/api/logs.ts

@@ -1,7 +1,7 @@
 /** @module @twilio-labs/serverless-api/dist/api */
 
 import debug from 'debug';
-import { GotClient, LogApiResource, LogList, Sid } from '../types';
+import { GotClient, LogApiResource, LogList, Sid, LogFilters } from '../types';
 import { getPaginatedResource } from './utils/pagination';
 
 const log = debug('twilio-serverless-api:logs');
@@ -30,6 +30,49 @@ export async function listLogResources(
   }
 }
 
+/**
+ * Calls the API to retrieve a list of all assets
+ *
+ * @param {Sid} environmentSid environment in which to get logs
+ * @param {Sid} serviceSid service to look for logs
+ * @param {GotClient} client API client
+ * @returns {Promise<LogApiResource[]>}
+ */
+export async function listOnePageLogResources(
+  environmentSid: Sid,
+  serviceSid: Sid,
+  client: GotClient,
+  filters: LogFilters
+): Promise<LogApiResource[]> {
+  const pageSize = filters.pageSize || 50;
+  const { functionSid, startDate, endDate, pageToken } = filters;
+  try {
+    let url = `/Services/${serviceSid}/Environments/${environmentSid}/Logs?PageSize=${pageSize}`;
+    if (typeof functionSid !== 'undefined') {
+      url += `&FunctionSid=${functionSid}`;
+    }
+    if (typeof startDate !== 'undefined') {
+      url += `&StartDate=${
+        startDate instanceof Date ? startDate.toISOString() : startDate
+      }`;
+    }
+    if (typeof endDate !== 'undefined') {
+      url += `&EndDate=${
+        endDate instanceof Date ? endDate.toISOString() : endDate
+      }`;
+    }
+    if (typeof pageToken !== 'undefined') {
+      url += `&PageToken=${pageToken}`;
+    }
+    const resp = await client.get(url);
+    const content = (resp.body as unknown) as LogList;
+    return content.logs as LogApiResource[];
+  } catch (err) {
+    log('%O', err);
+    throw err;
+  }
+}
+
 /**
  * Calls the API to retrieve a list of all assets
  *

src/client.ts

@@ -20,7 +20,12 @@ import {
   isEnvironmentSid,
   listEnvironments,
 } from './api/environments';
-import { getOrCreateFunctionResources, uploadFunction } from './api/functions';
+import {
+  getOrCreateFunctionResources,
+  uploadFunction,
+  isFunctionSid,
+  listFunctionResources,
+} from './api/functions';
 import { createService, findServiceSid, listServices } from './api/services';
 import {
   listVariablesForEnvironment,
@@ -37,9 +42,13 @@ import {
   GotClient,
   ListConfig,
   ListResult,
+  LogApiResource,
+  LogsConfig,
 } from './types';
 import { DeployStatus } from './types/consts';
 import { getListOfFunctionsAndAssets, SearchConfig } from './utils/fs';
+import { LogsStream } from './streams/logs';
+import { listOnePageLogResources } from './api/logs';
 
 const log = debug('twilio-serverless-api:client');
 
@@ -197,6 +206,75 @@ export class TwilioServerlessApiClient extends events.EventEmitter {
     return result;
   }
 
+  async getLogsStream(logsConfig: LogsConfig): Promise<LogsStream> {
+    let { serviceSid, environment, filterByFunction } = logsConfig;
+    if (!isEnvironmentSid(environment)) {
+      const environmentResource = await getEnvironmentFromSuffix(
+        environment,
+        serviceSid,
+        this.client
+      );
+      environment = environmentResource.sid;
+    }
+
+    if (filterByFunction && !isFunctionSid(filterByFunction)) {
+      const availableFunctions = await listFunctionResources(
+        serviceSid,
+        this.client
+      );
+      const foundFunction = availableFunctions.find(
+        fn => fn.friendly_name === filterByFunction
+      );
+      if (!foundFunction) {
+        throw new Error('Invalid Function Name or SID');
+      }
+      filterByFunction = foundFunction.sid;
+    }
+    const logsStream = new LogsStream(
+      environment,
+      serviceSid,
+      this.client,
+      logsConfig
+    );
+
+    return logsStream;
+  }
+
+  async getLogs(logsConfig: LogsConfig): Promise<LogApiResource[]> {
+    let { serviceSid, environment, filterByFunction } = logsConfig;
+    if (!isEnvironmentSid(environment)) {
+      const environmentResource = await getEnvironmentFromSuffix(
+        environment,
+        serviceSid,
+        this.client
+      );
+      environment = environmentResource.sid;
+    }
+
+    if (filterByFunction && !isFunctionSid(filterByFunction)) {
+      const availableFunctions = await listFunctionResources(
+        serviceSid,
+        this.client
+      );
+      const foundFunction = availableFunctions.find(
+        fn => fn.friendly_name === filterByFunction
+      );
+      if (!foundFunction) {
+        throw new Error('Invalid Function Name or SID');
+      }
+      filterByFunction = foundFunction.sid;
+    }
+
+    try {
+      return listOnePageLogResources(environment, serviceSid, this.client, {
+        pageSize: 50,
+        functionSid: filterByFunction,
+      });
+    } catch (e) {
+      throw e;
+    }
+  }
+
   /**
    * "Activates" a build by taking a specified build SID or a "source environment"
    * and activating the same build in the specified `environment`.

src/streams/logs.ts

@@ -0,0 +1,80 @@
+import { Readable } from 'stream';
+import { listOnePageLogResources } from '../api/logs';
+import { LogApiResource, Sid, GotClient } from '../types';
+import { LogsConfig } from '../types/logs';
+
+export class LogsStream extends Readable {
+  private _pollingFrequency: number;
+  private _interval: NodeJS.Timeout | undefined;
+  private _viewedSids: Set<Sid>;
+
+  constructor(
+    private environmentSid: Sid,
+    private serviceSid: Sid,
+    private client: GotClient,
+    private config: LogsConfig
+  ) {
+    super({ objectMode: true });
+    this._interval = undefined;
+    this._viewedSids = new Set();
+    this._pollingFrequency = config.pollingFrequency || 1000;
+  }
+
+  set pollingFrequency(frequency: number) {
+    this._pollingFrequency = frequency;
+    if (this.config.tail && this._interval) {
+      clearInterval(this._interval);
+      this._interval = setInterval(() => {
+        this._poll();
+      }, this._pollingFrequency);
+    }
+  }
+
+  async _poll() {
+    try {
+      const logs = await listOnePageLogResources(
+        this.environmentSid,
+        this.serviceSid,
+        this.client,
+        {
+          functionSid: this.config.filterByFunction,
+          pageSize: this.config.limit,
+        }
+      );
+      logs
+        .filter(log => !this._viewedSids.has(log.sid))
+        .reverse()
+        .forEach(log => {
+          this.push(log);
+        });
+      // Replace the set each time rather than adding to the set.
+      // This way the set is always the size of a page of logs and the next page
+      // will either overlap or not. This is instead of keeping an ever growing
+      // set of viewSids which would cause memory issues for long running log
+      // tails.
+      this._viewedSids = new Set(logs.map(log => log.sid));
+      if (!this.config.tail) {
+        this.push(null);
+      }
+    } catch (err) {
+      this.destroy(err);
+    }
+  }
+
+  _read() {
+    if (this.config.tail && !this._interval) {
+      this._interval = setInterval(() => {
+        this._poll();
+      }, this._pollingFrequency);
+    } else {
+      this._poll();
+    }
+  }
+
+  _destroy() {
+    if (this._interval) {
+      clearInterval(this._interval);
+      this._interval = undefined;
+    }
+  }
+}

src/types/index.ts

@@ -6,3 +6,4 @@ export * from './deploy';
 export * from './generic';
 export * from './list';
 export * from './serverless-api';
+export * from './logs';

src/types/logs.ts

@@ -0,0 +1,20 @@
+/** @module @twilio-labs/serverless-api */
+
+import { Sid } from './serverless-api';
+
+export type LogsConfig = {
+  serviceSid: Sid;
+  environment: string | Sid;
+  tail: boolean;
+  limit?: number;
+  filterByFunction?: string | Sid;
+  pollingFrequency?: number;
+};
+
+export type LogFilters = {
+  pageSize?: number;
+  functionSid?: Sid;
+  startDate?: string | Date;
+  endDate?: string | Date;
+  pageToken?: string;
+};