feat(logs): add logs fetching and streaming (#23)

* feat(logs): new function to only load one page of logs

* feat(logs): adds a streaming class for logs that polls for now

* feat(logs): adds convenience func, page size and function filter

* chore: updates package.json

Fixes reset-dependencies npm script and adds a watch script for continuous compilation.

* refactor(./src/streams/logs.ts): logsStream refactor

Should pass a client to LogsStream instead of credentials, as that's how everything else works. Access through the client should set the client.

Also, changed how filtering old logs out on subsequent polls works to contain memory.

* feat(./src/api/logs.ts): adds filters to listOnePageLogResources

Adds StartDate, EndDate amd PageToken as options to listOnePageLogResources.

* feat(client): adds getLogs function to client

so that twilio-run can get a single page of recent logs

* improvement(logs): turn filters for logs into an object argument

* improvement(logs): make polling frequency settable for log streams

* improvement(logs): in objectmode readable streams can push objects

This saves serializing and deserializing the data.

* improvement(logs): removed unused _buffer variable

Author: philnash

package.json

@@ -8,6 +8,7 @@
     "jest": "jest",
     "test": "run-s build:noemit jest",
     "build": "tsc",
+    "watch": "tsc --watch",
     "build:noemit": "tsc --noEmit",
     "postbuild": "npm shrinkwrap",
     "docs": "typedoc --options typedoc.json",
@@ -16,7 +17,7 @@
     "lint-staged": "lint-staged",
     "cm": "git-cz",
     "release": "HUSKY_SKIP_HOOKS=1 standard-version",
-    "reset-dependencies": "bash ./hooks/update-dependencies"
+    "reset-dependencies": "bash ./hooks/update-dependencies.sh"
   },
   "publishConfig": {
     "access": "public"

src/api/functions.ts

@@ -180,3 +180,14 @@ export async function uploadFunction(
   const version = await createFunctionVersion(fn, serviceSid, client);
   return version.sid;
 }
+
+/**
+ * Checks if a string is an function SID by checking its prefix and length
+ *
+ * @export
+ * @param {string} str the string to check
+ * @returns
+ */
+export function isFunctionSid(str: string) {
+  return str.startsWith('ZH') && str.length === 34;
+}

src/api/logs.ts

@@ -1,7 +1,7 @@
 /** @module @twilio-labs/serverless-api/dist/api */
 
 import debug from 'debug';
-import { GotClient, LogApiResource, LogList, Sid } from '../types';
+import { GotClient, LogApiResource, LogList, Sid, LogFilters } from '../types';
 import { getPaginatedResource } from './utils/pagination';
 
 const log = debug('twilio-serverless-api:logs');
@@ -30,6 +30,49 @@ export async function listLogResources(
   }
 }
 
+/**
+ * Calls the API to retrieve a list of all assets
+ *
+ * @param {Sid} environmentSid environment in which to get logs
+ * @param {Sid} serviceSid service to look for logs
+ * @param {GotClient} client API client
+ * @returns {Promise<LogApiResource[]>}
+ */
+export async function listOnePageLogResources(
+  environmentSid: Sid,
+  serviceSid: Sid,
+  client: GotClient,
+  filters: LogFilters
+): Promise<LogApiResource[]> {
+  const pageSize = filters.pageSize || 50;
+  const { functionSid, startDate, endDate, pageToken } = filters;
+  try {
+    let url = `/Services/${serviceSid}/Environments/${environmentSid}/Logs?PageSize=${pageSize}`;
+    if (typeof functionSid !== 'undefined') {
+      url += `&FunctionSid=${functionSid}`;
+    }
+    if (typeof startDate !== 'undefined') {
+      url += `&StartDate=${
+        startDate instanceof Date ? startDate.toISOString() : startDate
+      }`;
+    }
+    if (typeof endDate !== 'undefined') {
+      url += `&EndDate=${
+        endDate instanceof Date ? endDate.toISOString() : endDate
+      }`;
+    }
+    if (typeof pageToken !== 'undefined') {
+      url += `&PageToken=${pageToken}`;
+    }
+    const resp = await client.get(url);
+    const content = (resp.body as unknown) as LogList;
+    return content.logs as LogApiResource[];
+  } catch (err) {
+    log('%O', err);
+    throw err;
+  }
+}
+
 /**
  * Calls the API to retrieve a list of all assets
  *

src/client.ts

@@ -20,7 +20,12 @@ import {
   isEnvironmentSid,
   listEnvironments,
 } from './api/environments';
-import { getOrCreateFunctionResources, uploadFunction } from './api/functions';
+import {
+  getOrCreateFunctionResources,
+  uploadFunction,
+  isFunctionSid,
+  listFunctionResources,
+} from './api/functions';
 import { createService, findServiceSid, listServices } from './api/services';
 import {
   listVariablesForEnvironment,
@@ -37,9 +42,13 @@ import {
   GotClient,
   ListConfig,
   ListResult,
+  LogApiResource,
+  LogsConfig,
 } from './types';
 import { DeployStatus } from './types/consts';
 import { getListOfFunctionsAndAssets, SearchConfig } from './utils/fs';
+import { LogsStream } from './streams/logs';
+import { listOnePageLogResources } from './api/logs';
 
 const log = debug('twilio-serverless-api:client');
 
@@ -197,6 +206,75 @@ export class TwilioServerlessApiClient extends events.EventEmitter {
     return result;
   }
 
+  async getLogsStream(logsConfig: LogsConfig): Promise<LogsStream> {
+    let { serviceSid, environment, filterByFunction } = logsConfig;
+    if (!isEnvironmentSid(environment)) {
+      const environmentResource = await getEnvironmentFromSuffix(
+        environment,
+        serviceSid,
+        this.client
+      );
+      environment = environmentResource.sid;
+    }
+
+    if (filterByFunction && !isFunctionSid(filterByFunction)) {
+      const availableFunctions = await listFunctionResources(
+        serviceSid,
+        this.client
+      );
+      const foundFunction = availableFunctions.find(
+        fn => fn.friendly_name === filterByFunction
+      );
+      if (!foundFunction) {
+        throw new Error('Invalid Function Name or SID');
+      }
+      filterByFunction = foundFunction.sid;
+    }
+    const logsStream = new LogsStream(
+      environment,
+      serviceSid,
+      this.client,
+      logsConfig
+    );
+
+    return logsStream;
+  }
+
+  async getLogs(logsConfig: LogsConfig): Promise<LogApiResource[]> {
+    let { serviceSid, environment, filterByFunction } = logsConfig;
+    if (!isEnvironmentSid(environment)) {
+      const environmentResource = await getEnvironmentFromSuffix(
+        environment,
+        serviceSid,
+        this.client
+      );
+      environment = environmentResource.sid;
+    }
+
+    if (filterByFunction && !isFunctionSid(filterByFunction)) {
+      const availableFunctions = await listFunctionResources(
+        serviceSid,
+        this.client
+      );
+      const foundFunction = availableFunctions.find(
+        fn => fn.friendly_name === filterByFunction
+      );
+      if (!foundFunction) {
+        throw new Error('Invalid Function Name or SID');
+      }
+      filterByFunction = foundFunction.sid;
+    }
+
+    try {
+      return listOnePageLogResources(environment, serviceSid, this.client, {
+        pageSize: 50,
+        functionSid: filterByFunction,
+      });
+    } catch (e) {
+      throw e;
+    }
+  }
+
   /**
    * "Activates" a build by taking a specified build SID or a "source environment"
    * and activating the same build in the specified `environment`.

src/streams/logs.ts

@@ -0,0 +1,80 @@
+import { Readable } from 'stream';
+import { listOnePageLogResources } from '../api/logs';
+import { LogApiResource, Sid, GotClient } from '../types';
+import { LogsConfig } from '../types/logs';
+
+export class LogsStream extends Readable {
+  private _pollingFrequency: number;
+  private _interval: NodeJS.Timeout | undefined;
+  private _viewedSids: Set<Sid>;
+
+  constructor(
+    private environmentSid: Sid,
+    private serviceSid: Sid,
+    private client: GotClient,
+    private config: LogsConfig
+  ) {
+    super({ objectMode: true });
+    this._interval = undefined;
+    this._viewedSids = new Set();
+    this._pollingFrequency = config.pollingFrequency || 1000;
+  }
+
+  set pollingFrequency(frequency: number) {
+    this._pollingFrequency = frequency;
+    if (this.config.tail && this._interval) {
+      clearInterval(this._interval);
+      this._interval = setInterval(() => {
+        this._poll();
+      }, this._pollingFrequency);
+    }
+  }
+
+  async _poll() {
+    try {
+      const logs = await listOnePageLogResources(
+        this.environmentSid,
+        this.serviceSid,
+        this.client,
+        {
+          functionSid: this.config.filterByFunction,
+          pageSize: this.config.limit,
+        }
+      );
+      logs
+        .filter(log => !this._viewedSids.has(log.sid))
+        .reverse()
+        .forEach(log => {
+          this.push(log);
+        });
+      // Replace the set each time rather than adding to the set.
+      // This way the set is always the size of a page of logs and the next page
+      // will either overlap or not. This is instead of keeping an ever growing
+      // set of viewSids which would cause memory issues for long running log
+      // tails.
+      this._viewedSids = new Set(logs.map(log => log.sid));
+      if (!this.config.tail) {
+        this.push(null);
+      }
+    } catch (err) {
+      this.destroy(err);
+    }
+  }
+
+  _read() {
+    if (this.config.tail && !this._interval) {
+      this._interval = setInterval(() => {
+        this._poll();
+      }, this._pollingFrequency);
+    } else {
+      this._poll();
+    }
+  }
+
+  _destroy() {
+    if (this._interval) {
+      clearInterval(this._interval);
+      this._interval = undefined;
+    }
+  }
+}

src/types/index.ts

@@ -6,3 +6,4 @@ export * from './deploy';
 export * from './generic';
 export * from './list';
 export * from './serverless-api';
+export * from './logs';

src/types/logs.ts

@@ -0,0 +1,20 @@
+/** @module @twilio-labs/serverless-api */
+
+import { Sid } from './serverless-api';
+
+export type LogsConfig = {
+  serviceSid: Sid;
+  environment: string | Sid;
+  tail: boolean;
+  limit?: number;
+  filterByFunction?: string | Sid;
+  pollingFrequency?: number;
+};
+
+export type LogFilters = {
+  pageSize?: number;
+  functionSid?: Sid;
+  startDate?: string | Date;
+  endDate?: string | Date;
+  pageToken?: string;
+};