Add gzip support to getUrl in node (#1085).

ethers-io · Oct 7, 2020 · 65772a8 · 65772a8
1 parent a022093
commit 65772a8
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 4 deletions.
diff --git a/packages/web/src.ts/geturl.ts b/packages/web/src.ts/geturl.ts
@@ -2,9 +2,11 @@
 
 import http from "http";
 import https from "https";
+import { gunzipSync } from "zlib";
 import { parse } from "url"
 
-import { concat } from "@ethersproject/bytes";
+import { arrayify, concat } from "@ethersproject/bytes";
+import { shallowCopy } from "@ethersproject/properties";
 
 import type { GetUrlResponse, Options } from "./types";
 
@@ -38,6 +40,11 @@ function getResponse(request: http.ClientRequest): Promise<GetUrlResponse> {
             });
 
             resp.on("end", () => {
+                if (response.headers["content-encoding"] === "gzip") {
+                    //const size = response.body.length;
+                    response.body = arrayify(gunzipSync(response.body));
+                    //console.log("Delta:", response.body.length - size, Buffer.from(response.body).toString());
+                }
                 resolve(response);
             });
 
@@ -73,9 +80,13 @@ export async function getUrl(href: string, options?: Options): Promise<GetUrlRes
         path: (nonnull(url.pathname) + nonnull(url.search)),
 
         method: (options.method || "GET"),
-        headers: (options.headers || { }),
+        headers: shallowCopy(options.headers || { }),
     };
 
+    if (options.allowGzip) {
+        request.headers["accept-encoding"] = "gzip";
+    }
+
     let req: http.ClientRequest = null;
     switch (nonnull(url.protocol)) {
         case "http:":

diff --git a/packages/web/src.ts/index.ts b/packages/web/src.ts/index.ts
@@ -9,7 +9,7 @@ import { Logger } from "@ethersproject/logger";
 import { version } from "./_version";
 const logger = new Logger(version);
 
-import { getUrl, GetUrlResponse } from "./geturl";
+import { getUrl, GetUrlResponse, Options } from "./geturl";
 
 function staller(duration: number): Promise<void> {
     return new Promise((resolve) => {
@@ -43,6 +43,7 @@ export type ConnectionInfo = {
     password?: string,
 
     allowInsecureAuthentication?: boolean,
+    allowGzip?: boolean,
 
     throttleLimit?: number,
     throttleSlotInterval?: number;
@@ -100,7 +101,7 @@ export function _fetchData<T = Uint8Array>(connection: string | ConnectionInfo,
     let url: string = null;
 
     // @TODO: Allow ConnectionInfo to override some of these values
-    const options: any = {
+    const options: Options = {
         method: "GET",
     };
 
@@ -131,6 +132,8 @@ export function _fetchData<T = Uint8Array>(connection: string | ConnectionInfo,
             }
         }
 
+        options.allowGzip = !!connection.allowGzip;
+
         if (connection.user != null && connection.password != null) {
             if (url.substring(0, 6) !== "https:" && connection.allowInsecureAuthentication !== true) {
                 logger.throwError(

diff --git a/packages/web/src.ts/types.ts b/packages/web/src.ts/types.ts
@@ -9,6 +9,7 @@ export type GetUrlResponse = {
 
 export type Options = {
     method?: string,
+    allowGzip?: boolean;
     body?: Uint8Array
     headers?: { [ key: string] : string },
 };