Remove Buffer Usage (#801)

* Remove buffer usage in favor or Uint8Array and Dataview

* Add byteoffset when creating DataView

* Increase Indexer sleep time

* Add polyfill for base64 decoding and update examples

* Update examples to use utility function for Buffer functionality

* Revert smoke test

* Add a roundtrip test for sanity check

* Fix base64 encoding test

* Remove buffer in transaction

* Remove unnecessary casting

* Unify return type to Uint8Array

* Remove buffer usage in source code except client

* Standardize tests and add more utf-8 strings

* Add hex encoding and string encoding tests

* Fix some code broken by existing tests

* Update tests for edge bytes

* Export byte conversion utilities

* Fix imports to prefix algosdk

* Resolving merge conflicts

* Remove some generated types from stable release

* Fix node tests

* Try loadResource in browser

* Update buffer usage in client files

* Revert cucumber browser loadResource

* Fix browser tests

* Fix node tests

* Add alternative path for decoding composer responses

* Remove browser buffer dependency, fix cucumber tests

* Revert cucumber node index.js file to use buffer again

* Fix some conversion methods in err messages

* Update src/encoding/binarydata.ts

Co-authored-by: Eric Warehime <eric.warehime@gmail.com>

* Run prettier on commit suggestion

* Add toString conversion for buffers

* Update tests/cucumber/browser/test.js

Co-authored-by: Jason Paulos <jasonpaulos@users.noreply.github.com>

---------

Co-authored-by: Eric Warehime <eric.warehime@gmail.com>
Co-authored-by: Jason Paulos <jasonpaulos@users.noreply.github.com>

.eslintrc.js

@@ -15,16 +15,6 @@ module.exports = {
   },
   plugins: ['@typescript-eslint', 'eslint-plugin-tsdoc'],
   rules: {
-    'no-restricted-globals': [
-      'error',
-      {
-        // This is to ensure that we use the 'buffer' package in the browser. In Node it doesn't
-        // make a difference.
-        name: 'Buffer',
-        message:
-          "Explictly import Buffer with `import { Buffer } from 'buffer'`",
-      },
-    ],
     'no-constant-condition': ['error', { checkLoops: false }],
     'no-restricted-syntax': ['error', 'LabeledStatement', 'WithStatement'],
     'no-plusplus': ['error', { allowForLoopAfterthoughts: true }],

examples/asa.ts

@@ -49,7 +49,7 @@ async function main() {
   console.log(`Asset Params: ${JSON.stringify(assetInfo.params)}`);
   // example: ASSET_INFO
 
-  await new Promise((f) => setTimeout(f, 2000)); // sleep to ensure indexer is caught up
+  await new Promise((f) => setTimeout(f, 5000)); // sleep to ensure indexer is caught up
 
   // example: INDEXER_LOOKUP_ASSET
   const indexer = getLocalIndexerClient();

examples/codec.ts

@@ -3,7 +3,7 @@
 /* eslint-disable no-promise-executor-return */
 /* eslint-disable no-console */
 import algosdk from '../src';
-import { getLocalAlgodClient, getLocalAccounts } from './utils';
+import { getLocalAccounts, getLocalAlgodClient } from './utils';
 
 async function main() {
   const client = getLocalAlgodClient();

examples/lsig.ts

@@ -3,7 +3,7 @@
 /* eslint-disable no-promise-executor-return */
 /* eslint-disable no-console */
 import algosdk from '../src';
-import { getLocalAlgodClient, getLocalAccounts } from './utils';
+import { getLocalAccounts, getLocalAlgodClient } from './utils';
 
 async function main() {
   const client = getLocalAlgodClient();

package.json

@@ -22,7 +22,6 @@
   },
   "dependencies": {
     "algo-msgpack-with-bigint": "^2.1.1",
-    "buffer": "^6.0.3",
     "cross-fetch": "^4.0.0-alpha.5",
     "hi-base32": "^0.5.1",
     "js-sha256": "^0.9.0",

src/bid.ts

@@ -1,4 +1,3 @@
-import { Buffer } from 'buffer';
 import * as address from './encoding/address';
 import * as encoding from './encoding/encoding';
 import * as nacl from './nacl/naclWrappers';
@@ -27,7 +26,7 @@ export type BidOptions = Omit<
  * */
 export default class Bid implements BidStorageStructure {
   name = 'Bid';
-  tag = Buffer.from([97, 66]); // "aB"
+  tag = Uint8Array.from([97, 66]); // "aB"
 
   bidderKey: Address;
   bidAmount: number;
@@ -67,23 +66,23 @@ export default class Bid implements BidStorageStructure {
   // eslint-disable-next-line camelcase
   get_obj_for_encoding() {
     return {
-      bidder: Buffer.from(this.bidderKey.publicKey),
+      bidder: this.bidderKey.publicKey,
       cur: this.bidAmount,
       price: this.maxPrice,
       id: this.bidID,
-      auc: Buffer.from(this.auctionKey.publicKey),
+      auc: this.auctionKey.publicKey,
       aid: this.auctionID,
     };
   }
 
   signBid(sk: Uint8Array) {
     const encodedMsg = encoding.encode(this.get_obj_for_encoding());
-    const toBeSigned = Buffer.from(utils.concatArrays(this.tag, encodedMsg));
+    const toBeSigned = utils.concatArrays(this.tag, encodedMsg);
     const sig = nacl.sign(toBeSigned, sk);
 
     // construct signed message
     const sBid = {
-      sig: Buffer.from(sig),
+      sig,
       bid: this.get_obj_for_encoding(),
     };
 

src/client/client.ts

@@ -1,4 +1,3 @@
-import { Buffer } from 'buffer';
 import * as utils from '../utils/utils';
 import {
   BaseHTTPClient,
@@ -151,10 +150,10 @@ export default class HTTPClient {
       return new Uint8Array(0); // empty Uint8Array
     }
     if (requestHeaders['content-type'] === 'application/json') {
-      return new Uint8Array(Buffer.from(JSON.stringify(data)));
+      return new TextEncoder().encode(JSON.stringify(data));
     }
     if (typeof data === 'string') {
-      return new Uint8Array(Buffer.from(data));
+      return new TextEncoder().encode(data);
     }
     if (data instanceof Uint8Array) {
       return data;
@@ -179,7 +178,7 @@ export default class HTTPClient {
     let text;
 
     if (format !== 'application/msgpack') {
-      text = (body && Buffer.from(body).toString()) || '';
+      text = (body && new TextDecoder().decode(body)) || '';
     }
 
     if (parseBody && format === 'application/json') {

src/client/kmd.ts

@@ -1,7 +1,11 @@
-import { Buffer } from 'buffer';
-import ServiceClient from './v2/serviceClient';
+import {
+  base64ToBytes,
+  bytesToBase64,
+  coerceToBytes,
+} from '../encoding/binarydata';
 import * as txn from '../transaction';
 import { CustomTokenHeader, KMDTokenHeader } from './urlTokenBaseHTTPClient';
+import ServiceClient from './v2/serviceClient';
 
 export default class Kmd extends ServiceClient {
   constructor(
@@ -50,7 +54,7 @@ export default class Kmd extends ServiceClient {
       wallet_name: walletName,
       wallet_driver_name: walletDriverName,
       wallet_password: walletPassword,
-      master_derivation_key: Buffer.from(walletMDK).toString('base64'),
+      master_derivation_key: bytesToBase64(walletMDK),
     };
     const res = await this.c.post('/v1/wallet', req);
     return res.body;
@@ -157,10 +161,7 @@ export default class Kmd extends ServiceClient {
     };
     const res = await this.c.post('/v1/master-key/export', req);
     return {
-      master_derivation_key: Buffer.from(
-        res.body.master_derivation_key,
-        'base64'
-      ),
+      master_derivation_key: base64ToBytes(res.body.master_derivation_key),
     };
   }
 
@@ -174,7 +175,7 @@ export default class Kmd extends ServiceClient {
   async importKey(walletHandle: string, secretKey: Uint8Array) {
     const req = {
       wallet_handle_token: walletHandle,
-      private_key: Buffer.from(secretKey).toString('base64'),
+      private_key: bytesToBase64(secretKey),
     };
     const res = await this.c.post('/v1/key/import', req);
     return res.body;
@@ -195,7 +196,7 @@ export default class Kmd extends ServiceClient {
       wallet_password: walletPassword,
     };
     const res = await this.c.post('/v1/key/export', req);
-    return { private_key: Buffer.from(res.body.private_key, 'base64') };
+    return { private_key: base64ToBytes(res.body.private_key) };
   }
 
   /**
@@ -266,12 +267,12 @@ export default class Kmd extends ServiceClient {
     const req = {
       wallet_handle_token: walletHandle,
       wallet_password: walletPassword,
-      transaction: Buffer.from(tx.toByte()).toString('base64'),
+      transaction: bytesToBase64(tx.toByte()),
     };
     const res = await this.c.post('/v1/transaction/sign', req);
 
     if (res.status === 200) {
-      return Buffer.from(res.body.signed_transaction, 'base64');
+      return base64ToBytes(res.body.signed_transaction);
     }
     return res.body;
   }
@@ -293,17 +294,18 @@ export default class Kmd extends ServiceClient {
     publicKey: Uint8Array | string
   ) {
     const tx = txn.instantiateTxnIfNeeded(transaction);
+    const pk = coerceToBytes(publicKey);
 
     const req = {
       wallet_handle_token: walletHandle,
       wallet_password: walletPassword,
-      transaction: Buffer.from(tx.toByte()).toString('base64'),
-      public_key: Buffer.from(publicKey).toString('base64'),
+      transaction: bytesToBase64(tx.toByte()),
+      public_key: bytesToBase64(pk),
     };
     const res = await this.c.post('/v1/transaction/sign', req);
 
     if (res.status === 200) {
-      return Buffer.from(res.body.signed_transaction, 'base64');
+      return base64ToBytes(res.body.signed_transaction);
     }
     return res.body;
   }
@@ -389,10 +391,11 @@ export default class Kmd extends ServiceClient {
     partial: string
   ) {
     const tx = txn.instantiateTxnIfNeeded(transaction);
+    const pubkey = coerceToBytes(pk);
     const req = {
       wallet_handle_token: walletHandle,
-      transaction: Buffer.from(tx.toByte()).toString('base64'),
-      public_key: Buffer.from(pk).toString('base64'),
+      transaction: bytesToBase64(tx.toByte()),
+      public_key: bytesToBase64(pubkey),
       partial_multisig: partial,
       wallet_password: pw,
     };

src/client/urlTokenBaseHTTPClient.ts

@@ -1,4 +1,3 @@
-import { Buffer } from 'buffer';
 import { fetch, Response, Headers } from 'cross-fetch';
 import {
   BaseHTTPClient,
@@ -118,7 +117,7 @@ export class URLTokenBaseHTTPClient implements BaseHTTPClient {
     try {
       body = new Uint8Array(await res.arrayBuffer());
       const decoded: Record<string, any> = JSON.parse(
-        Buffer.from(body).toString()
+        new TextDecoder().decode(body)
       );
       if (decoded.message) {
         bodyErrorMessage = decoded.message;

src/client/v2/algod/compile.ts

@@ -1,6 +1,6 @@
-import { Buffer } from 'buffer';
-import JSONRequest from '../jsonrequest';
+import { coerceToBytes } from '../../../encoding/binarydata';
 import HTTPClient from '../../client';
+import JSONRequest from '../jsonrequest';
 
 /**
  * Sets the default header (if not previously set)
@@ -42,7 +42,7 @@ export default class Compile extends JSONRequest {
     const txHeaders = setHeaders(headers);
     const res = await this.c.post(
       this.path(),
-      Buffer.from(this.source),
+      coerceToBytes(this.source),
       txHeaders,
       this.query
     );