From 1f033ca85165f1e39e8fae035a8d2c3c2453afa8 Mon Sep 17 00:00:00 2001 From: ChrisBoehmCA Date: Tue, 2 Apr 2019 14:05:36 -0400 Subject: [PATCH 1/7] introduce streaming REST API functions Signed-off-by: ChrisBoehmCA --- packages/io/__tests__/IO.test.ts | 18 +++ .../__tests__/__snapshots__/IO.test.ts.snap | 4 + packages/io/src/IO.ts | 23 ++++ .../client/AbstractRestClient.test.ts | 60 ++++++++- .../rest/src/client/AbstractRestClient.ts | 119 ++++++++++++++---- packages/rest/src/client/RestClient.ts | 70 +++++++++++ 6 files changed, 266 insertions(+), 28 deletions(-) diff --git a/packages/io/__tests__/IO.test.ts b/packages/io/__tests__/IO.test.ts index edf0441af..a54b853cb 100644 --- a/packages/io/__tests__/IO.test.ts +++ b/packages/io/__tests__/IO.test.ts @@ -317,6 +317,24 @@ describe("IO tests", () => { expect(error.message).toMatchSnapshot(); }); + it("should get an error for no input on createReadStream", () => { + let error; + try { + IO.createReadStream(" "); + } catch (thrownError) { + error = thrownError; + } + expect(error.message).toMatchSnapshot(); + }); + it("should get an error for no input on createWriteStream", () => { + let error; + try { + IO.createWriteStream(" "); + } catch (thrownError) { + error = thrownError; + } + expect(error.message).toMatchSnapshot(); + }); it("should get an error for no input on createFileSync", () => { let error; try { diff --git a/packages/io/__tests__/__snapshots__/IO.test.ts.snap b/packages/io/__tests__/__snapshots__/IO.test.ts.snap index 054f3713b..78cbd4e8a 100644 --- a/packages/io/__tests__/__snapshots__/IO.test.ts.snap +++ b/packages/io/__tests__/__snapshots__/IO.test.ts.snap @@ -8,6 +8,10 @@ exports[`IO tests should get an error for no input on createDirsSyncFromFilePath exports[`IO tests should get an error for no input on createFileSync 1`] = `"Expect Error: Required parameter 'file' must not be blank"`; +exports[`IO tests should get an error for no input on createReadStream 1`] = `"Expect Error: Required parameter 'file' must not be blank"`; + +exports[`IO tests should get an error for no input on createWriteStream 1`] = `"Expect Error: Required parameter 'file' must not be blank"`; + exports[`IO tests should get an error for no input on deleteDir 1`] = `"Expect Error: Required parameter 'dir' must not be blank"`; exports[`IO tests should get an error for no input on deleteFile 1`] = `"Expect Error: Required parameter 'file' must not be blank"`; diff --git a/packages/io/src/IO.ts b/packages/io/src/IO.ts index 649ff07c0..41e5b2e5f 100644 --- a/packages/io/src/IO.ts +++ b/packages/io/src/IO.ts @@ -16,6 +16,7 @@ import { isNullOrUndefined } from "util"; import { ImperativeReject } from "../../interfaces"; import { ImperativeError } from "../../error"; import { ImperativeExpect } from "../../expect"; +import { Readable, Writable } from "stream"; const mkdirp = require("mkdirp"); @@ -231,6 +232,28 @@ export class IO { } } + /** + * Create a Node.js Readable stream from a file + * @param file - the file from which to create a read stream + * @return Buffer - the content of the file + * @memberof IO + */ + public static createReadStream(file: string): Readable { + ImperativeExpect.toBeDefinedAndNonBlank(file, "file"); + return fs.createReadStream(file, {autoClose: true}); + } + + /** + * Create a Node.js Readable stream from a file + * @param file - the file from which to create a read stream + * @return Buffer - the content of the file + * @memberof IO + */ + public static createWriteStream(file: string): Writable { + ImperativeExpect.toBeDefinedAndNonBlank(file, "file"); + return fs.createWriteStream(file, {autoClose: true}); + } + /** * Process a string so that its line endings are operating system * appropriate before you save it to disk diff --git a/packages/rest/__tests__/client/AbstractRestClient.test.ts b/packages/rest/__tests__/client/AbstractRestClient.test.ts index 52cb70b5c..f35549893 100644 --- a/packages/rest/__tests__/client/AbstractRestClient.test.ts +++ b/packages/rest/__tests__/client/AbstractRestClient.test.ts @@ -133,7 +133,7 @@ describe("AbstractRestClient tests", () => { }); (https.request as any) = requestFnc; - const headers = [{"My-Header": "value is here"}]; + const headers: any = [{"My-Header": "value is here"}]; const payload: any = {"my payload object": "hello"}; let error; try { @@ -416,4 +416,62 @@ describe("AbstractRestClient tests", () => { expect(httpsRequestFnc).toBeCalled(); expect(httpRequestFnc).not.toBeCalled(); }); + + it("should not error when streaming data", async () => { + + interface IPayload { + data: string; + } + + const fakeResponseStream: any = { + write: jest.fn(), + on: jest.fn(), + end: jest.fn() + }; + const fakeRequestStream: any = { + on: jest.fn((eventName: string, callback: any) => { + + }), + }; + const emitter = new MockHttpRequestResponse(); + const requestFnc = jest.fn((options, callback) => { + ProcessUtils.nextTick(async () => { + + const newEmit = new MockHttpRequestResponse(); + callback(newEmit); + + await ProcessUtils.nextTick(() => { + newEmit.emit("data", Buffer.from("{\"newData\":", "utf8")); + }); + + await ProcessUtils.nextTick(() => { + newEmit.emit("data", Buffer.from("\"response data\"}", "utf8")); + }); + + await ProcessUtils.nextTick(() => { + newEmit.emit("end"); + }); + }); + + return emitter; + }); + + (https.request as any) = requestFnc; + + await RestClient.putStreamed(new Session({ + hostname: "test", + }), "/resource", [Headers.APPLICATION_JSON], fakeResponseStream, fakeRequestStream); + + await RestClient.postStreamed(new Session({ + hostname: "test", + }), "/resource", [Headers.APPLICATION_JSON], fakeResponseStream, fakeRequestStream); + + await RestClient.getStreamed(new Session({ + hostname: "test", + }), "/resource", [Headers.APPLICATION_JSON], fakeResponseStream); + + await RestClient.deleteStreamed(new Session({ + hostname: "test", + }), "/resource", [Headers.APPLICATION_JSON], fakeResponseStream); + }); }); diff --git a/packages/rest/src/client/AbstractRestClient.ts b/packages/rest/src/client/AbstractRestClient.ts index df7abd258..3c69387d9 100644 --- a/packages/rest/src/client/AbstractRestClient.ts +++ b/packages/rest/src/client/AbstractRestClient.ts @@ -26,6 +26,7 @@ import * as path from "path"; import { IRestClientError } from "./doc/IRestClientError"; import { RestClientError } from "./RestClientError"; import { PerfTiming } from "@zowe/perf-timing"; +import { Readable, Writable } from "stream"; export type RestClientResolve = (data: string) => void; @@ -118,6 +119,24 @@ export abstract class AbstractRestClient { */ protected mWriteData: any; + /** + * Stream for incoming response data from the server. + * If specified, response data will not be buffered + * @private + * @type {Writable} + * @memberof AbstractRestClient + */ + protected mResponseStream: Writable; + + /** + * stream for outgoing request data to the server + * @private + * @type {Writable} + * @memberof AbstractRestClient + */ + protected mRequestStream: Readable; + + /** * Creates an instance of AbstractRestClient. * @param {AbstractSession} mSession - representing connection to this api @@ -135,10 +154,13 @@ export abstract class AbstractRestClient { * @param {string} request - REST request type GET|PUT|POST|DELETE * @param {any[]} reqHeaders - option headers to include with request * @param {any} writeData - data to write on this REST request + * @param responseStream - stream for incoming response data from the server. If specified, response data will not be buffered + * @param requestStream - stream for outgoing request data to the server * @throws if the request gets a status code outside of the 200 range * or other connection problems occur (e.g. connection refused) */ - public performRest(resource: string, request: HTTP_VERB, reqHeaders?: any[], writeData?: any): Promise { + public performRest(resource: string, request: HTTP_VERB, reqHeaders?: any[], writeData?: any, + responseStream?: Writable, requestStream?: Readable): Promise { return new Promise((resolve: RestClientResolve, reject: ImperativeReject) => { const timingApi = PerfTiming.api; @@ -153,6 +175,8 @@ export abstract class AbstractRestClient { this.mRequest = request; this.mReqHeaders = reqHeaders; this.mWriteData = writeData; + this.mRequestStream = requestStream; + this.mResponseStream = responseStream; // got a new promise this.mResolve = resolve; @@ -160,16 +184,17 @@ export abstract class AbstractRestClient { ImperativeExpect.toBeDefinedAndNonBlank(resource, "resource"); ImperativeExpect.toBeDefinedAndNonBlank(request, "request"); + ImperativeExpect.toBeEqual(requestStream != null && writeData != null, false, + "You cannot specify both writeData and writeStream"); const options = this.buildOptions(resource, request, reqHeaders); /** * Perform the actual http request */ - let clientRequest; + let clientRequest: https.ClientRequest | http.ClientRequest; if (this.session.ISession.protocol === AbstractSession.HTTPS_PROTOCOL) { clientRequest = https.request(options, this.requestHandler.bind(this)); - } - else if (this.session.ISession.protocol === AbstractSession.HTTP_PROTOCOL) { + } else if (this.session.ISession.protocol === AbstractSession.HTTP_PROTOCOL) { clientRequest = http.request(options, this.requestHandler.bind(this)); } @@ -203,9 +228,27 @@ export abstract class AbstractRestClient { })); }); - // always end the request - clientRequest.end(); - + if (requestStream != null) { + // if the user requested streaming write of data to the request, + // write the data chunk by chunk to the server + requestStream.on("data", (data: Buffer) => { + clientRequest.write(data); + }); + requestStream.on("error", (streamError: any) => { + reject(this.populateError({ + msg: "Error reading requestStream", + causeErrors: streamError, + source: "client" + })); + }); + requestStream.on("end", () => { + // finish the request + clientRequest.end(); + }); + } else { + // otherwise we're done with the request + clientRequest.end(); + } if (PerfTiming.isEnabled) { // Marks point END timingApi.mark("END_PERFORM_REST"); @@ -225,8 +268,7 @@ export abstract class AbstractRestClient { protected appendHeaders(headers: any[] | undefined): any[] { if (headers == null) { return []; - } - else { + } else { return headers; } } @@ -350,6 +392,16 @@ export abstract class AbstractRestClient { } } + if (this.mResponseStream != null) { + this.mResponseStream.on("error", (streamError: any) => { + this.mReject(this.populateError({ + msg: "Error writing to responseStream", + causeErrors: streamError, + source: "client" + })); + }); + } + /** * Invoke any onData method whenever data becomes available */ @@ -375,7 +427,16 @@ export abstract class AbstractRestClient { */ private onData(respData: Buffer): void { this.log.trace("Data chunk received..."); - this.mData = Buffer.concat([this.mData, respData]); + if (this.requestFailure || this.mResponseStream == null) { + // buffer the data if we are not streaming + // or if we encountered an error, since the rest client + // relies on any JSON error to be in the this.dataString field + this.mData = Buffer.concat([this.mData, respData]); + } else { + this.log.debug("Streaming data chunk of length " + respData.length + " to response stream"); + // write the chunk to the response stream if requested + this.mResponseStream.write(respData); + } } /** @@ -386,6 +447,10 @@ export abstract class AbstractRestClient { */ private onEnd(): void { this.log.debug("onEnd() called for rest client %s", this.constructor.name); + if (this.mResponseStream) { + this.log.debug("Ending response stream"); + this.mResponseStream.end(); + } if (this.requestFailure) { // Reject the promise with an error const errorCode = this.response == null ? undefined : this.response.statusCode; @@ -429,9 +494,9 @@ export abstract class AbstractRestClient { let payloadDetails: string = this.mWriteData + ""; try { headerDetails = JSON.stringify(this.mReqHeaders); - payloadDetails = inspect(this.mWriteData, { depth: null }); + payloadDetails = inspect(this.mWriteData, {depth: null}); } catch (stringifyError) { - this.log.error("Error encountered trying to parse details for REST request error:\n %s", inspect(stringifyError, { depth: null })); + this.log.error("Error encountered trying to parse details for REST request error:\n %s", inspect(stringifyError, {depth: null})); } // Populate the "relevant" fields - caller will have the session, so @@ -450,20 +515,20 @@ export abstract class AbstractRestClient { // Construct a formatted details message const detailMessage: string = - ((finalError.source === "client") ? - `HTTP(S) client encountered an error. Request could not be initiated to host.\n` + - `Review connection details (host, port) and ensure correctness.` - : - `HTTP(S) error status "${finalError.httpStatus}" received.\n` + - `Review request details (resource, base path, credentials, payload) and ensure correctness.`) + - "\n" + - "\nHost: " + finalError.host + - "\nPort: " + finalError.port + - "\nBase Path: " + finalError.basePath + - "\nResource: " + finalError.resource + - "\nRequest: " + finalError.request + - "\nHeaders: " + headerDetails + - "\nPayload: " + payloadDetails; + ((finalError.source === "client") ? + `HTTP(S) client encountered an error. Request could not be initiated to host.\n` + + `Review connection details (host, port) and ensure correctness.` + : + `HTTP(S) error status "${finalError.httpStatus}" received.\n` + + `Review request details (resource, base path, credentials, payload) and ensure correctness.`) + + "\n" + + "\nHost: " + finalError.host + + "\nPort: " + finalError.port + + "\nBase Path: " + finalError.basePath + + "\nResource: " + finalError.resource + + "\nRequest: " + finalError.request + + "\nHeaders: " + headerDetails + + "\nPayload: " + payloadDetails; finalError.additionalDetails = detailMessage; // Allow implementation to modify the error as necessary @@ -472,7 +537,7 @@ export abstract class AbstractRestClient { const processedError = this.processError(error); if (processedError != null) { this.log.debug("Error was processed by overridden processError method in RestClient %s", this.constructor.name); - finalError = { ...finalError, ...processedError }; + finalError = {...finalError, ...processedError}; } // Return the error object diff --git a/packages/rest/src/client/RestClient.ts b/packages/rest/src/client/RestClient.ts index eb2ac0a4d..60af3379f 100644 --- a/packages/rest/src/client/RestClient.ts +++ b/packages/rest/src/client/RestClient.ts @@ -14,6 +14,7 @@ import { RestConstants } from "./RestConstants"; import { HTTP_VERB } from "./types/HTTPVerb"; import { AbstractRestClient } from "./AbstractRestClient"; import { JSONUtils } from "../../../utilities"; +import { Readable, Writable } from "stream"; /** * Class to handle http(s) requests, build headers, collect data, report status codes, and header responses @@ -229,6 +230,75 @@ export class RestClient extends AbstractRestClient { return new this(session).performRest(resource, HTTP_VERB.DELETE, reqHeaders); } + /** + * REST HTTP GET operation + * @static + * @param {AbstractSession} session - representing connection to this api + * @param {string} resource - URI for which this request should go against + * @param {any} reqHeaders - headers to include in the REST request + * @param responseStream - the stream to which the response data will be written + * @returns {Promise} - empty string - data is not buffered for streamed requests + * @throws if the request gets a status code outside of the 200 range + * or other connection problems occur (e.g. connection refused) + * @memberof RestClient + */ + public static getStreamed(session: AbstractSession, resource: string, reqHeaders: any[] = [], + responseStream: Writable): Promise { + return new this(session).performRest(resource, HTTP_VERB.GET, reqHeaders, undefined, responseStream); + } + + /** + * REST HTTP PUT operation + * @static + * @param {AbstractSession} session - representing connection to this api + * @param {string} resource - URI for which this request should go against + * @param {object[]} reqHeaders - headers to include in the REST request + * @param {any} responseStream - stream to which the response data will be written + * @param {any} requestStream - stream from which payload data will be read + * @returns {Promise} - empty string - data is not buffered for streamed requests + * @throws if the request gets a status code outside of the 200 range + * or other connection problems occur (e.g. connection refused) + * @memberof RestClient + */ + public static putStreamed(session: AbstractSession, resource: string, reqHeaders: any[] = [], + responseStream: Writable, requestStream: Readable): Promise { + return new this(session).performRest(resource, HTTP_VERB.PUT, reqHeaders, undefined, responseStream, requestStream); + } + + /** + * REST HTTP POST operation + * @static + * @param {AbstractSession} session - representing connection to this api + * @param {string} resource - URI for which this request should go against + * @param {object[]} reqHeaders - headers to include in the REST request + * @param {any} responseStream - stream to which the response data will be written + * @param {any} requestStream - stream from which payload data will be read + * @returns {Promise} - empty string - data is not buffered for streamed requests + * @throws if the request gets a status code outside of the 200 range + * or other connection problems occur (e.g. connection refused) + * @memberof RestClient + */ + public static postStreamed(session: AbstractSession, resource: string, reqHeaders: any[] = [], + responseStream: Writable, requestStream: Readable): Promise { + return new this(session).performRest(resource, HTTP_VERB.POST, reqHeaders, undefined, responseStream, requestStream); + } + + /** + * REST HTTP DELETE operation + * @static + * @param {AbstractSession} session - representing connection to this api + * @param {string} resource - URI for which this request should go against + * @param {any} reqHeaders - headers to include in the REST request + * @param {any} responseStream - stream to which the response data will be written + * @returns {Promise} - empty string - data is not buffered for streamed requests + * @throws if the request gets a status code outside of the 200 range + * or other connection problems occur (e.g. connection refused) + * @memberof RestClient + */ + public static deleteStreamed(session: AbstractSession, resource: string, reqHeaders: any[] = [], responseStream: Writable): Promise { + return new this(session).performRest(resource, HTTP_VERB.DELETE, reqHeaders); + } + /** * Helper method to return an indicator for whether or not a URI contains a query string. * @static From 4f3c6766be40e639ef19fb4ea7affd06ffbe6b40 Mon Sep 17 00:00:00 2001 From: ChrisBoehmCA Date: Wed, 3 Apr 2019 13:30:58 -0400 Subject: [PATCH 2/7] add ability to normalize new lines from rest client Signed-off-by: ChrisBoehmCA --- .../client/AbstractRestClient.test.ts | 8 ++ .../rest/src/client/AbstractRestClient.ts | 36 +++++++- packages/rest/src/client/RestClient.ts | 89 ++++++++++++++++--- 3 files changed, 118 insertions(+), 15 deletions(-) diff --git a/packages/rest/__tests__/client/AbstractRestClient.test.ts b/packages/rest/__tests__/client/AbstractRestClient.test.ts index f35549893..4de90a19e 100644 --- a/packages/rest/__tests__/client/AbstractRestClient.test.ts +++ b/packages/rest/__tests__/client/AbstractRestClient.test.ts @@ -466,6 +466,14 @@ describe("AbstractRestClient tests", () => { hostname: "test", }), "/resource", [Headers.APPLICATION_JSON], fakeResponseStream, fakeRequestStream); + await RestClient.putStreamedRequestOnly(new Session({ + hostname: "test", + }), "/resource", [Headers.APPLICATION_JSON], fakeRequestStream); + + await RestClient.postStreamedRequestOnly(new Session({ + hostname: "test", + }), "/resource", [Headers.APPLICATION_JSON], fakeRequestStream); + await RestClient.getStreamed(new Session({ hostname: "test", }), "/resource", [Headers.APPLICATION_JSON], fakeResponseStream); diff --git a/packages/rest/src/client/AbstractRestClient.ts b/packages/rest/src/client/AbstractRestClient.ts index 3c69387d9..d34c89332 100644 --- a/packages/rest/src/client/AbstractRestClient.ts +++ b/packages/rest/src/client/AbstractRestClient.ts @@ -80,13 +80,31 @@ export abstract class AbstractRestClient { protected mResponse: any; /** - * Indicate if payload data is JSON to be stringified before writing + * Indicates if payload data is JSON to be stringified before writing * @private * @type {boolean} * @memberof AbstractRestClient */ protected mIsJson: boolean; + /** + * Indicates if request data should have its newlines normalized to /n before sending + * each chunk to the server + * @private + * @type {boolean} + * @memberof AbstractRestClient + */ + protected mNormalizeRequestNewlines: boolean; + + /** + * Indicates if response data should have its newlines normalized for the current platform + * (\r\n for windows, otherwise \n) + * @private + * @type {boolean} + * @memberof AbstractRestClient + */ + protected mNormalizeResponseNewlines: boolean; + /** * Save resource * @private @@ -156,11 +174,16 @@ export abstract class AbstractRestClient { * @param {any} writeData - data to write on this REST request * @param responseStream - stream for incoming response data from the server. If specified, response data will not be buffered * @param requestStream - stream for outgoing request data to the server + * @param normalizeResponseNewLines - streaming only - true if you want newlines to be \r\n on windows + * when receiving data from the server to responseStream. Don't set this for binary responses + * @param normalizeRequestNewLines - streaming only - true if you want \r\n to be replaced with \n when sending + * data to the server from requestStream. Don't set this for binary requests * @throws if the request gets a status code outside of the 200 range * or other connection problems occur (e.g. connection refused) */ public performRest(resource: string, request: HTTP_VERB, reqHeaders?: any[], writeData?: any, - responseStream?: Writable, requestStream?: Readable): Promise { + responseStream?: Writable, requestStream?: Readable, + normalizeResponseNewLines?: boolean, normalizeRequestNewLines?: boolean): Promise { return new Promise((resolve: RestClientResolve, reject: ImperativeReject) => { const timingApi = PerfTiming.api; @@ -177,6 +200,8 @@ export abstract class AbstractRestClient { this.mWriteData = writeData; this.mRequestStream = requestStream; this.mResponseStream = responseStream; + this.mNormalizeRequestNewlines = normalizeRequestNewLines; + this.mNormalizeResponseNewlines = normalizeResponseNewLines; // got a new promise this.mResolve = resolve; @@ -232,9 +257,15 @@ export abstract class AbstractRestClient { // if the user requested streaming write of data to the request, // write the data chunk by chunk to the server requestStream.on("data", (data: Buffer) => { + this.log.debug("Writing data chunk of length %d from requestStream to clientRequest", data.byteLength); + if (this.mNormalizeRequestNewlines) { + this.log.debug("Normalizing new lines in request chunk to \\n"); + data = Buffer.from(data.toString().replace(/\r?\n/g, "\n")); + } clientRequest.write(data); }); requestStream.on("error", (streamError: any) => { + this.log.error("Error encountered reading requestStream: " + streamError); reject(this.populateError({ msg: "Error reading requestStream", causeErrors: streamError, @@ -242,6 +273,7 @@ export abstract class AbstractRestClient { })); }); requestStream.on("end", () => { + this.log.debug("Finished reading requestStream"); // finish the request clientRequest.end(); }); diff --git a/packages/rest/src/client/RestClient.ts b/packages/rest/src/client/RestClient.ts index 60af3379f..c96eb2efd 100644 --- a/packages/rest/src/client/RestClient.ts +++ b/packages/rest/src/client/RestClient.ts @@ -231,56 +231,114 @@ export class RestClient extends AbstractRestClient { } /** - * REST HTTP GET operation + * REST HTTP GET operation - streaming the response to a writable stream * @static * @param {AbstractSession} session - representing connection to this api * @param {string} resource - URI for which this request should go against * @param {any} reqHeaders - headers to include in the REST request * @param responseStream - the stream to which the response data will be written - * @returns {Promise} - empty string - data is not buffered for streamed requests + * @param normalizeResponseNewLines - streaming only - true if you want newlines to be \r\n on windows + * when receiving data from the server to responseStream. Don't set this for binary responses + * @returns {Promise} - empty string - data is not buffered for this request * @throws if the request gets a status code outside of the 200 range * or other connection problems occur (e.g. connection refused) * @memberof RestClient */ public static getStreamed(session: AbstractSession, resource: string, reqHeaders: any[] = [], - responseStream: Writable): Promise { - return new this(session).performRest(resource, HTTP_VERB.GET, reqHeaders, undefined, responseStream); + responseStream: Writable, + normalizeResponseNewLines?: boolean): Promise { + return new this(session).performRest(resource, HTTP_VERB.GET, reqHeaders, undefined, responseStream, + undefined, normalizeResponseNewLines); } /** - * REST HTTP PUT operation + * REST HTTP PUT operation with streamed response and request * @static * @param {AbstractSession} session - representing connection to this api * @param {string} resource - URI for which this request should go against * @param {object[]} reqHeaders - headers to include in the REST request * @param {any} responseStream - stream to which the response data will be written * @param {any} requestStream - stream from which payload data will be read + * @param normalizeResponseNewLines - streaming only - true if you want newlines to be \r\n on windows + * when receiving data from the server to responseStream. Don't set this for binary responses + * @param normalizeRequestNewLines - streaming only - true if you want \r\n to be replaced with \n when sending + * data to the server from requestStream. Don't set this for binary requests * @returns {Promise} - empty string - data is not buffered for streamed requests * @throws if the request gets a status code outside of the 200 range * or other connection problems occur (e.g. connection refused) * @memberof RestClient */ public static putStreamed(session: AbstractSession, resource: string, reqHeaders: any[] = [], - responseStream: Writable, requestStream: Readable): Promise { - return new this(session).performRest(resource, HTTP_VERB.PUT, reqHeaders, undefined, responseStream, requestStream); + responseStream: Writable, requestStream: Readable, + normalizeResponseNewLines?: boolean, normalizeRequestNewLines?: boolean): Promise { + return new this(session).performRest(resource, HTTP_VERB.PUT, reqHeaders, undefined, responseStream, requestStream, + normalizeResponseNewLines, normalizeRequestNewLines); } /** - * REST HTTP POST operation + * REST HTTP PUT operation with only streamed request, buffers response data and returns it + * @static + * @param {AbstractSession} session - representing connection to this api + * @param {string} resource - URI for which this request should go against + * @param {object[]} reqHeaders - headers to include in the REST request + * @param {any} requestStream - stream from which payload data will be read + + * @param normalizeRequestNewLines - streaming only - true if you want \r\n to be replaced with \n when sending + * data to the server from requestStream. Don't set this for binary requests + * @returns {Promise} - string of the response + * @throws if the request gets a status code outside of the 200 range + * or other connection problems occur (e.g. connection refused) + * @memberof RestClient + */ + public static putStreamedRequestOnly(session: AbstractSession, resource: string, reqHeaders: any[] = [], + requestStream: Readable, + normalizeRequestNewLines?: boolean): Promise { + return new this(session).performRest(resource, HTTP_VERB.PUT, reqHeaders, undefined, undefined, requestStream, + undefined, normalizeRequestNewLines); + } + + /** + * REST HTTP POST operation streaming both the request and the response * @static * @param {AbstractSession} session - representing connection to this api * @param {string} resource - URI for which this request should go against * @param {object[]} reqHeaders - headers to include in the REST request * @param {any} responseStream - stream to which the response data will be written * @param {any} requestStream - stream from which payload data will be read - * @returns {Promise} - empty string - data is not buffered for streamed requests + * @param normalizeResponseNewLines - streaming only - true if you want newlines to be \r\n on windows + * when receiving data from the server to responseStream. Don't set this for binary responses + * @param normalizeRequestNewLines - streaming only - true if you want \r\n to be replaced with \n when sending + * data to the server from requestStream. Don't set this for binary requests + * @returns {Promise} - empty string - data is not buffered for this request * @throws if the request gets a status code outside of the 200 range * or other connection problems occur (e.g. connection refused) * @memberof RestClient */ public static postStreamed(session: AbstractSession, resource: string, reqHeaders: any[] = [], - responseStream: Writable, requestStream: Readable): Promise { - return new this(session).performRest(resource, HTTP_VERB.POST, reqHeaders, undefined, responseStream, requestStream); + responseStream: Writable, requestStream: Readable, + normalizeResponseNewLines?: boolean, normalizeRequestNewLines?: boolean): Promise { + return new this(session).performRest(resource, HTTP_VERB.POST, reqHeaders, undefined, responseStream, requestStream, + normalizeResponseNewLines, normalizeRequestNewLines); + } + + /** + * REST HTTP POST operation, streaming only the request and not the response + * @static + * @param {AbstractSession} session - representing connection to this api + * @param {string} resource - URI for which this request should go against + * @param {object[]} reqHeaders - headers to include in the REST request + * @param {any} requestStream - stream from which payload data will be read + * @param normalizeRequestNewLines - streaming only - true if you want \r\n to be replaced with \n when sending + * data to the server from requestStream. Don't set this for binary requests + * @returns {Promise} - string of the response + * @throws if the request gets a status code outside of the 200 range + * or other connection problems occur (e.g. connection refused) + * @memberof RestClient + */ + public static postStreamedRequestOnly(session: AbstractSession, resource: string, reqHeaders: any[] = [], + requestStream: Readable, normalizeRequestNewLines?: boolean): Promise { + return new this(session).performRest(resource, HTTP_VERB.POST, reqHeaders, undefined, undefined, requestStream, + undefined, normalizeRequestNewLines); } /** @@ -290,13 +348,18 @@ export class RestClient extends AbstractRestClient { * @param {string} resource - URI for which this request should go against * @param {any} reqHeaders - headers to include in the REST request * @param {any} responseStream - stream to which the response data will be written + * @param normalizeResponseNewLines - streaming only - true if you want newlines to be \r\n on windows + * when receiving data from the server to responseStream. Don't set this for binary responses * @returns {Promise} - empty string - data is not buffered for streamed requests * @throws if the request gets a status code outside of the 200 range * or other connection problems occur (e.g. connection refused) * @memberof RestClient */ - public static deleteStreamed(session: AbstractSession, resource: string, reqHeaders: any[] = [], responseStream: Writable): Promise { - return new this(session).performRest(resource, HTTP_VERB.DELETE, reqHeaders); + public static deleteStreamed(session: AbstractSession, resource: string, reqHeaders: any[] = [], responseStream: Writable, + normalizeResponseNewLines?: boolean): Promise { + return new this(session).performRest(resource, HTTP_VERB.DELETE, reqHeaders, + undefined, responseStream, undefined, normalizeResponseNewLines + ); } /** From 3aef9fe39c73a816b7ff5a48dbe8faaa56ba696b Mon Sep 17 00:00:00 2001 From: ChrisBoehmCA Date: Thu, 4 Apr 2019 10:29:29 -0400 Subject: [PATCH 3/7] add progress bar task updating to rest client (optional) Signed-off-by: ChrisBoehmCA --- package.json | 56 +++++++++------ .../rest/src/client/AbstractRestClient.ts | 69 ++++++++++++++++++- packages/rest/src/client/RestClient.ts | 37 ++++++---- 3 files changed, 128 insertions(+), 34 deletions(-) diff --git a/package.json b/package.json index e32b871eb..6fba60aa0 100644 --- a/package.json +++ b/package.json @@ -4,30 +4,34 @@ "description": "framework for building configurable CLIs", "author": "Broadcom", "license": "EPL-2.0", - "repository": { + "repository": { "type": "git", "url": "https://github.com/zowe/imperative.git" }, - "keywords": [ + "keywords": [ "CLI", "framework", "zowe" ], - "pkg": { - "scripts": [ + "pkg": { + "scripts": [ "lib/**/*.js", "lib/auth/*.js" ], - "assets": [ + "assets": [ "../../npm/node_modules/node-gyp/**/*", "../../npm/lib/" ] }, - "files": ["lib"], - "publishConfig": {"registry": "https://gizaartifactory.jfrog.io/gizaartifactory/api/npm/npm-local-release/"}, + "files": [ + "lib" + ], + "publishConfig": { + "registry": "https://gizaartifactory.jfrog.io/gizaartifactory/api/npm/npm-local-release/" + }, "main": "lib/index.js", "typings": "lib/index.d.ts", - "scripts": { + "scripts": { "audit:public": "npm audit --registry https://registry.npmjs.org/", "build:packages": "gulp build", "build": "gulp build", @@ -43,7 +47,7 @@ "lint:tests": "tslint --format stylish -c ./tslint-tests.json \"**/__tests__/**/*.ts\"", "watch": "gulp watch" }, - "dependencies": { + "dependencies": { "@types/yargs": "8.0.2", "@zowe/perf-timing": "^1.0.3-alpha.201902201448", "chalk": "2.1.0", @@ -70,7 +74,7 @@ "yamljs": "0.3.0", "yargs": "8.0.2" }, - "devDependencies": { + "devDependencies": { "@types/body-parser": "1.16.4", "@types/chai": "4.0.1", "@types/chai-string": "1.1.30", @@ -121,24 +125,36 @@ "uuid": "3.2.1", "yargs-parser": "9.0.2" }, - "engines": {"node": ">=6.0.0"}, - "jest-junit": {"output": "./__tests__/__results__/junit/junit.xml"}, - "jestSonar": {"reportPath": "__tests__/__results__/jest-sonar"}, - "jest-stare": { + "engines": { + "node": ">=6.0.0" + }, + "jest-junit": { + "output": "./__tests__/__results__/junit/junit.xml" + }, + "jestSonar": { + "reportPath": "__tests__/__results__/jest-sonar" + }, + "jest-stare": { "resultDir": "__tests__/__results__/jest-stare", - "additionalResultsProcessors": [ + "additionalResultsProcessors": [ "jest-junit", "jest-sonar-reporter" ], "coverageLink": "../unit/coverage/lcov-report/index.html" }, - "jest": { - "globals": {"ts-jest": {"diagnostics": false}}, - "moduleFileExtensions": [ + "jest": { + "globals": { + "ts-jest": { + "diagnostics": false + } + }, + "moduleFileExtensions": [ "ts", "js" ], "testEnvironment": "node", - "transform": {".(ts)": "ts-jest"} + "transform": { + ".(ts)": "ts-jest" + } } -} \ No newline at end of file +} diff --git a/packages/rest/src/client/AbstractRestClient.ts b/packages/rest/src/client/AbstractRestClient.ts index d34c89332..1aa11fd65 100644 --- a/packages/rest/src/client/AbstractRestClient.ts +++ b/packages/rest/src/client/AbstractRestClient.ts @@ -27,6 +27,9 @@ import { IRestClientError } from "./doc/IRestClientError"; import { RestClientError } from "./RestClientError"; import { PerfTiming } from "@zowe/perf-timing"; import { Readable, Writable } from "stream"; +import { IO } from "../../../io"; +import { ITaskWithStatus, TaskProgress, TaskStage } from "../../../operations"; +import { TextUtils } from "../../../utilities"; export type RestClientResolve = (data: string) => void; @@ -79,6 +82,15 @@ export abstract class AbstractRestClient { */ protected mResponse: any; + /** + * If we get a response containing a Content-Length header, + * it is saved here + * @private + * @type {number} + * @memberof AbstractRestClient + */ + protected mContentLength: number; + /** * Indicates if payload data is JSON to be stringified before writing * @private @@ -154,6 +166,22 @@ export abstract class AbstractRestClient { */ protected mRequestStream: Readable; + /** + * Task used to display progress bars or other user feedback mechanisms + * Automatically updated if task is specified and streams are provided for upload/download + * @private + * @type {ITaskWithStatus} + * @memberof AbstractRestClient + */ + protected mTask: ITaskWithStatus; + + /** + * Bytes received from the server response so far + * @private + * @type {ITaskWithStatus} + * @memberof AbstractRestClient + */ + protected mBytesReceived: number = 0; /** * Creates an instance of AbstractRestClient. @@ -178,12 +206,14 @@ export abstract class AbstractRestClient { * when receiving data from the server to responseStream. Don't set this for binary responses * @param normalizeRequestNewLines - streaming only - true if you want \r\n to be replaced with \n when sending * data to the server from requestStream. Don't set this for binary requests + * @param task - task that will automatically be updated to report progress of upload or download to user * @throws if the request gets a status code outside of the 200 range * or other connection problems occur (e.g. connection refused) */ public performRest(resource: string, request: HTTP_VERB, reqHeaders?: any[], writeData?: any, responseStream?: Writable, requestStream?: Readable, - normalizeResponseNewLines?: boolean, normalizeRequestNewLines?: boolean): Promise { + normalizeResponseNewLines?: boolean, normalizeRequestNewLines?: boolean, + task?: ITaskWithStatus): Promise { return new Promise((resolve: RestClientResolve, reject: ImperativeReject) => { const timingApi = PerfTiming.api; @@ -202,6 +232,7 @@ export abstract class AbstractRestClient { this.mResponseStream = responseStream; this.mNormalizeRequestNewlines = normalizeRequestNewLines; this.mNormalizeResponseNewlines = normalizeResponseNewLines; + this.mTask = task; // got a new promise this.mResolve = resolve; @@ -256,12 +287,17 @@ export abstract class AbstractRestClient { if (requestStream != null) { // if the user requested streaming write of data to the request, // write the data chunk by chunk to the server + let bytesUploaded = 0; requestStream.on("data", (data: Buffer) => { this.log.debug("Writing data chunk of length %d from requestStream to clientRequest", data.byteLength); if (this.mNormalizeRequestNewlines) { this.log.debug("Normalizing new lines in request chunk to \\n"); data = Buffer.from(data.toString().replace(/\r?\n/g, "\n")); } + if (this.mTask != null) { + bytesUploaded += data.byteLength; + this.mTask.statusMessage = TextUtils.formatMessage("Uploading %d B", bytesUploaded, this.mContentLength); + } clientRequest.write(data); }); requestStream.on("error", (streamError: any) => { @@ -422,6 +458,14 @@ export abstract class AbstractRestClient { this.session.storeCookie(this.response.headers[RestConstants.PROP_COOKIE]); } } + if (Headers.CONTENT_LENGTH in this.response.headers) { + this.mContentLength = this.response.headers[Headers.CONTENT_LENGTH]; + this.log.debug("Content length of response is: " + this.mContentLength); + } + if (Headers.CONTENT_LENGTH.toLowerCase() in this.response.headers) { + this.mContentLength = this.response.headers[Headers.CONTENT_LENGTH.toLowerCase()]; + this.log.debug("Content length of response is: " + this.mContentLength); + } } if (this.mResponseStream != null) { @@ -459,6 +503,7 @@ export abstract class AbstractRestClient { */ private onData(respData: Buffer): void { this.log.trace("Data chunk received..."); + this.mBytesReceived += respData.byteLength; if (this.requestFailure || this.mResponseStream == null) { // buffer the data if we are not streaming // or if we encountered an error, since the rest client @@ -466,6 +511,22 @@ export abstract class AbstractRestClient { this.mData = Buffer.concat([this.mData, respData]); } else { this.log.debug("Streaming data chunk of length " + respData.length + " to response stream"); + if (this.mNormalizeResponseNewlines) { + this.log.debug("Normalizing new lines in data chunk to operating system appropriate line endings"); + respData = Buffer.from(IO.processNewlines(respData.toString())); + } + if (this.mTask != null) { + // update the progress task if provided by the requester + if (this.mContentLength != null) { + this.mTask.percentComplete = Math.floor(TaskProgress.ONE_HUNDRED_PERCENT * + (this.mBytesReceived / this.mContentLength)); + this.mTask.statusMessage = TextUtils.formatMessage("Downloading %d of %d B", + this.mBytesReceived, this.mContentLength); + } else { + this.mTask.statusMessage = TextUtils.formatMessage("Downloaded %d B", + this.mBytesReceived); + } + } // write the chunk to the response stream if requested this.mResponseStream.write(respData); } @@ -479,7 +540,11 @@ export abstract class AbstractRestClient { */ private onEnd(): void { this.log.debug("onEnd() called for rest client %s", this.constructor.name); - if (this.mResponseStream) { + if (this.mTask != null) { + this.mTask.percentComplete = TaskProgress.ONE_HUNDRED_PERCENT; + this.mTask.stageName = TaskStage.COMPLETE; + } + if (this.mResponseStream != null) { this.log.debug("Ending response stream"); this.mResponseStream.end(); } diff --git a/packages/rest/src/client/RestClient.ts b/packages/rest/src/client/RestClient.ts index c96eb2efd..182ef8691 100644 --- a/packages/rest/src/client/RestClient.ts +++ b/packages/rest/src/client/RestClient.ts @@ -15,6 +15,7 @@ import { HTTP_VERB } from "./types/HTTPVerb"; import { AbstractRestClient } from "./AbstractRestClient"; import { JSONUtils } from "../../../utilities"; import { Readable, Writable } from "stream"; +import { ITaskWithStatus } from "../../../operations"; /** * Class to handle http(s) requests, build headers, collect data, report status codes, and header responses @@ -239,6 +240,7 @@ export class RestClient extends AbstractRestClient { * @param responseStream - the stream to which the response data will be written * @param normalizeResponseNewLines - streaming only - true if you want newlines to be \r\n on windows * when receiving data from the server to responseStream. Don't set this for binary responses + * @param {ITaskWithStatus} task - task used to update the user on the progress of their request * @returns {Promise} - empty string - data is not buffered for this request * @throws if the request gets a status code outside of the 200 range * or other connection problems occur (e.g. connection refused) @@ -246,9 +248,10 @@ export class RestClient extends AbstractRestClient { */ public static getStreamed(session: AbstractSession, resource: string, reqHeaders: any[] = [], responseStream: Writable, - normalizeResponseNewLines?: boolean): Promise { + normalizeResponseNewLines?: boolean, + task?: ITaskWithStatus): Promise { return new this(session).performRest(resource, HTTP_VERB.GET, reqHeaders, undefined, responseStream, - undefined, normalizeResponseNewLines); + undefined, normalizeResponseNewLines, undefined, task); } /** @@ -263,6 +266,7 @@ export class RestClient extends AbstractRestClient { * when receiving data from the server to responseStream. Don't set this for binary responses * @param normalizeRequestNewLines - streaming only - true if you want \r\n to be replaced with \n when sending * data to the server from requestStream. Don't set this for binary requests + * @param {ITaskWithStatus} task - task used to update the user on the progress of their request * @returns {Promise} - empty string - data is not buffered for streamed requests * @throws if the request gets a status code outside of the 200 range * or other connection problems occur (e.g. connection refused) @@ -270,9 +274,10 @@ export class RestClient extends AbstractRestClient { */ public static putStreamed(session: AbstractSession, resource: string, reqHeaders: any[] = [], responseStream: Writable, requestStream: Readable, - normalizeResponseNewLines?: boolean, normalizeRequestNewLines?: boolean): Promise { + normalizeResponseNewLines?: boolean, normalizeRequestNewLines?: boolean, + task?: ITaskWithStatus): Promise { return new this(session).performRest(resource, HTTP_VERB.PUT, reqHeaders, undefined, responseStream, requestStream, - normalizeResponseNewLines, normalizeRequestNewLines); + normalizeResponseNewLines, normalizeRequestNewLines, task); } /** @@ -285,6 +290,7 @@ export class RestClient extends AbstractRestClient { * @param normalizeRequestNewLines - streaming only - true if you want \r\n to be replaced with \n when sending * data to the server from requestStream. Don't set this for binary requests + * @param {ITaskWithStatus} task - task used to update the user on the progress of their request * @returns {Promise} - string of the response * @throws if the request gets a status code outside of the 200 range * or other connection problems occur (e.g. connection refused) @@ -292,9 +298,10 @@ export class RestClient extends AbstractRestClient { */ public static putStreamedRequestOnly(session: AbstractSession, resource: string, reqHeaders: any[] = [], requestStream: Readable, - normalizeRequestNewLines?: boolean): Promise { + normalizeRequestNewLines?: boolean, + task?: ITaskWithStatus): Promise { return new this(session).performRest(resource, HTTP_VERB.PUT, reqHeaders, undefined, undefined, requestStream, - undefined, normalizeRequestNewLines); + undefined, normalizeRequestNewLines, task); } /** @@ -309,6 +316,7 @@ export class RestClient extends AbstractRestClient { * when receiving data from the server to responseStream. Don't set this for binary responses * @param normalizeRequestNewLines - streaming only - true if you want \r\n to be replaced with \n when sending * data to the server from requestStream. Don't set this for binary requests + * @param {ITaskWithStatus} task - task used to update the user on the progress of their request * @returns {Promise} - empty string - data is not buffered for this request * @throws if the request gets a status code outside of the 200 range * or other connection problems occur (e.g. connection refused) @@ -316,9 +324,10 @@ export class RestClient extends AbstractRestClient { */ public static postStreamed(session: AbstractSession, resource: string, reqHeaders: any[] = [], responseStream: Writable, requestStream: Readable, - normalizeResponseNewLines?: boolean, normalizeRequestNewLines?: boolean): Promise { + normalizeResponseNewLines?: boolean, normalizeRequestNewLines?: boolean, + task?: ITaskWithStatus): Promise { return new this(session).performRest(resource, HTTP_VERB.POST, reqHeaders, undefined, responseStream, requestStream, - normalizeResponseNewLines, normalizeRequestNewLines); + normalizeResponseNewLines, normalizeRequestNewLines, task); } /** @@ -330,15 +339,17 @@ export class RestClient extends AbstractRestClient { * @param {any} requestStream - stream from which payload data will be read * @param normalizeRequestNewLines - streaming only - true if you want \r\n to be replaced with \n when sending * data to the server from requestStream. Don't set this for binary requests + * @param {ITaskWithStatus} task - task used to update the user on the progress of their request * @returns {Promise} - string of the response * @throws if the request gets a status code outside of the 200 range * or other connection problems occur (e.g. connection refused) * @memberof RestClient */ public static postStreamedRequestOnly(session: AbstractSession, resource: string, reqHeaders: any[] = [], - requestStream: Readable, normalizeRequestNewLines?: boolean): Promise { + requestStream: Readable, normalizeRequestNewLines?: boolean, + task?: ITaskWithStatus): Promise { return new this(session).performRest(resource, HTTP_VERB.POST, reqHeaders, undefined, undefined, requestStream, - undefined, normalizeRequestNewLines); + undefined, normalizeRequestNewLines, task); } /** @@ -348,6 +359,7 @@ export class RestClient extends AbstractRestClient { * @param {string} resource - URI for which this request should go against * @param {any} reqHeaders - headers to include in the REST request * @param {any} responseStream - stream to which the response data will be written + * @param {ITaskWithStatus} task - task used to update the user on the progress of their request * @param normalizeResponseNewLines - streaming only - true if you want newlines to be \r\n on windows * when receiving data from the server to responseStream. Don't set this for binary responses * @returns {Promise} - empty string - data is not buffered for streamed requests @@ -356,9 +368,10 @@ export class RestClient extends AbstractRestClient { * @memberof RestClient */ public static deleteStreamed(session: AbstractSession, resource: string, reqHeaders: any[] = [], responseStream: Writable, - normalizeResponseNewLines?: boolean): Promise { + normalizeResponseNewLines?: boolean, + task?: ITaskWithStatus): Promise { return new this(session).performRest(resource, HTTP_VERB.DELETE, reqHeaders, - undefined, responseStream, undefined, normalizeResponseNewLines + undefined, responseStream, undefined, normalizeResponseNewLines, undefined, task ); } From 99435b1c41602dd8ca69f7e3c30637273ce009fb Mon Sep 17 00:00:00 2001 From: ChrisBoehmCA Date: Thu, 4 Apr 2019 10:35:46 -0400 Subject: [PATCH 4/7] fix lint Signed-off-by: ChrisBoehmCA --- packages/rest/src/client/RestClient.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/rest/src/client/RestClient.ts b/packages/rest/src/client/RestClient.ts index 182ef8691..bd3765f40 100644 --- a/packages/rest/src/client/RestClient.ts +++ b/packages/rest/src/client/RestClient.ts @@ -287,7 +287,6 @@ export class RestClient extends AbstractRestClient { * @param {string} resource - URI for which this request should go against * @param {object[]} reqHeaders - headers to include in the REST request * @param {any} requestStream - stream from which payload data will be read - * @param normalizeRequestNewLines - streaming only - true if you want \r\n to be replaced with \n when sending * data to the server from requestStream. Don't set this for binary requests * @param {ITaskWithStatus} task - task used to update the user on the progress of their request From 358d0a55265c41ab2433329f201dca567632e1c9 Mon Sep 17 00:00:00 2001 From: ChrisBoehmCA Date: Thu, 4 Apr 2019 10:43:00 -0400 Subject: [PATCH 5/7] fix lint Signed-off-by: ChrisBoehmCA --- packages/rest/__tests__/client/AbstractRestClient.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/rest/__tests__/client/AbstractRestClient.test.ts b/packages/rest/__tests__/client/AbstractRestClient.test.ts index 4de90a19e..82fb0a6ba 100644 --- a/packages/rest/__tests__/client/AbstractRestClient.test.ts +++ b/packages/rest/__tests__/client/AbstractRestClient.test.ts @@ -430,7 +430,7 @@ describe("AbstractRestClient tests", () => { }; const fakeRequestStream: any = { on: jest.fn((eventName: string, callback: any) => { - + // do nothing }), }; const emitter = new MockHttpRequestResponse(); From bf28da54ebc8a55324c2530aa8cd36dae4cfad24 Mon Sep 17 00:00:00 2001 From: ChrisBoehmCA Date: Thu, 4 Apr 2019 10:51:32 -0400 Subject: [PATCH 6/7] check if headers are null before searching for content length Signed-off-by: ChrisBoehmCA --- packages/rest/src/client/AbstractRestClient.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/rest/src/client/AbstractRestClient.ts b/packages/rest/src/client/AbstractRestClient.ts index 1aa11fd65..1bee51d13 100644 --- a/packages/rest/src/client/AbstractRestClient.ts +++ b/packages/rest/src/client/AbstractRestClient.ts @@ -458,13 +458,15 @@ export abstract class AbstractRestClient { this.session.storeCookie(this.response.headers[RestConstants.PROP_COOKIE]); } } - if (Headers.CONTENT_LENGTH in this.response.headers) { - this.mContentLength = this.response.headers[Headers.CONTENT_LENGTH]; - this.log.debug("Content length of response is: " + this.mContentLength); - } - if (Headers.CONTENT_LENGTH.toLowerCase() in this.response.headers) { - this.mContentLength = this.response.headers[Headers.CONTENT_LENGTH.toLowerCase()]; - this.log.debug("Content length of response is: " + this.mContentLength); + if (this.response.headers != null) { + if (Headers.CONTENT_LENGTH in this.response.headers) { + this.mContentLength = this.response.headers[Headers.CONTENT_LENGTH]; + this.log.debug("Content length of response is: " + this.mContentLength); + } + if (Headers.CONTENT_LENGTH.toLowerCase() in this.response.headers) { + this.mContentLength = this.response.headers[Headers.CONTENT_LENGTH.toLowerCase()]; + this.log.debug("Content length of response is: " + this.mContentLength); + } } } From 24f7e614069b9d2540dafee0c1e56bc78819a2e0 Mon Sep 17 00:00:00 2001 From: ChrisBoehmCA Date: Wed, 10 Apr 2019 09:22:50 -0400 Subject: [PATCH 7/7] increment progress bar if content-length is unknown up to 90% Signed-off-by: ChrisBoehmCA --- packages/rest/src/client/AbstractRestClient.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/rest/src/client/AbstractRestClient.ts b/packages/rest/src/client/AbstractRestClient.ts index 1bee51d13..45dac73f1 100644 --- a/packages/rest/src/client/AbstractRestClient.ts +++ b/packages/rest/src/client/AbstractRestClient.ts @@ -296,7 +296,12 @@ export abstract class AbstractRestClient { } if (this.mTask != null) { bytesUploaded += data.byteLength; - this.mTask.statusMessage = TextUtils.formatMessage("Uploading %d B", bytesUploaded, this.mContentLength); + this.mTask.statusMessage = TextUtils.formatMessage("Uploading %d B", bytesUploaded); + if (this.mTask.percentComplete < TaskProgress.NINETY_PERCENT) { + // we don't know how far along we are but increment the percentage to + // show we are making progress + this.mTask.percentComplete++; + } } clientRequest.write(data); }); @@ -525,8 +530,13 @@ export abstract class AbstractRestClient { this.mTask.statusMessage = TextUtils.formatMessage("Downloading %d of %d B", this.mBytesReceived, this.mContentLength); } else { - this.mTask.statusMessage = TextUtils.formatMessage("Downloaded %d B", + this.mTask.statusMessage = TextUtils.formatMessage("Downloaded %d of ? B", this.mBytesReceived); + if (this.mTask.percentComplete < TaskProgress.NINETY_PERCENT) { + // we don't know how far along we are but increment the percentage to + // show that we are making progress + this.mTask.percentComplete++; + } } } // write the chunk to the response stream if requested