Merge pull request #1248 from Borewit/update-from-commonjs-to-esm

Major update, including multiple changes:

.github/dependabot.yml

@@ -1,21 +1,12 @@
 version: 2
 updates:
 
-  # CommonJS (CJS)
   - package-ecosystem: npm
     directory: "/"
     schedule:
       interval: weekly
-    open-pull-requests-limit: 10
+    open-pull-requests-limit: 15
     labels:
       - dependencies
-      - CJS
     target-branch: master
     versioning-strategy: increase
-    ignore:
-      - dependency-name: strtok3
-        versions:
-          - "7.x" # Pure ESM module
-      - dependency-name: mocha
-        versions:
-          - "10.x"

.github/workflows/nodejs-ci.yml

@@ -7,7 +7,6 @@ jobs:
   build:
 
     runs-on: ubuntu-latest
-    environment: AWS
 
     # These permissions are needed to interact with GitHub's OIDC Token endpoint.
     permissions:
@@ -16,25 +15,22 @@ jobs:
 
     steps:
 
-      # Fail fast
-      - name: Test AWS OIDC autentication
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-region: eu-west-2
-          role-to-assume: arn:aws:iam::970661486861:role/github-actions-role
-          role-session-name: GithubActionS3
+#      # Fail fast
+#      - name: Test AWS OIDC authentication
+#        uses: aws-actions/configure-aws-credentials@v1
+#        with:
+#          aws-region: eu-west-2
+#          role-to-assume: arn:aws:iam::970661486861:role/github-actions-role
+#          role-session-name: GithubActionS3
 
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
-      - uses: actions/setup-node@v1
+      - uses: actions/setup-node@v4
         with:
-          node-version: 16.x
-
-      - name: Install production dependencies, check node engine compatiblity
-        run: yarn install --production=true
+          node-version: 20.x
 
       - name: Install development dependencies
-        run: yarn install --production=false --ignore-engines
+        run: yarn install
 
       - name: Build & Code analysis
         run: yarn run lint
@@ -43,7 +39,7 @@ jobs:
         run: yarn run build
 
       - name: Upload build
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: build
           path: |
@@ -56,12 +52,12 @@ jobs:
   test:
 
     runs-on: ubuntu-latest
-    environment: AWS
+    environment: AWS S3
     needs: build
 
     strategy:
       matrix:
-        node-version: [14.x, 16.x, 18.x]
+        node-version: [16.x, 18.x, 20.x, 22.x]
 
     # These permissions are needed to interact with GitHub's OIDC Token endpoint.
     permissions:
@@ -70,28 +66,31 @@ jobs:
 
     steps:
 
-      - name: Configure AWS OIDC Session
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-region: eu-west-2
-          role-to-assume: arn:aws:iam::970661486861:role/github-actions-role
-          role-session-name: GithubActionS3
+#      - name: Configure AWS OIDC Session
+#        uses: aws-actions/configure-aws-credentials@v1
+#        with:
+#          aws-region: eu-west-2
+#          role-to-assume: arn:aws:iam::970661486861:role/github-actions-role
+#          role-session-name: GithubActionS3
 
       - name: 'Checkout the repository'
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Test with Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
 
       - name: Install dependencies
         run: yarn install --ignore-engines
 
       - name: Download build
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           name: build
 
       - name: Test with Node.js ${{ matrix.node-version }}
         run: yarn run test
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.mocharc.json

@@ -0,0 +1,7 @@
+{
+  "extension": ["ts", "tsx"],
+  "watch-files": ["lib/**/*.ts", "test/**/*.ts"],
+  "spec": ["test/*.ts"],
+  "loader": ["ts-node/esm"],
+  "extensions": ["ts", "tsx"]
+}

README.md

@@ -27,13 +27,17 @@ To configure AWS client authentication see [Configuration and credential file se
 Determine file type (based on it's content) from a file stored Amazon S3 cloud:
 ```js
 const FileType = require('file-type');
-const { S3Client } = require('@aws-sdk/client-s3');
+import { fromEnv } from '@aws-sdk/credential-providers';
+import { S3Client } from '@aws-sdk/client-s3';
 const { makeTokenizer } = require('@tokenizer/s3');
 
 (async () => {
 
   // Initialize S3 client
-  const s3 = new S3Client({});
+  const s3 = new S3Client({
+    region: 'eu-west-2',
+    credentials: fromEnv(),
+  });
 
   // Initialize  S3 tokenizer
   const s3Tokenizer = await makeTokenizer(s3, {

lib/index.ts

@@ -1,7 +1,7 @@
 import { S3Client, GetObjectRequest, GetObjectCommand } from '@aws-sdk/client-s3';
 import { parseContentRange, tokenizer } from '@tokenizer/range';
-import * as strtok3 from 'strtok3/lib/core';
-import { S3Request } from './s3-request';
+import { fromStream, type ITokenizer } from 'strtok3';
+import { S3Request } from './s3-request.js';
 import { Readable } from 'stream';
 
 export interface IS3Options {
@@ -18,15 +18,17 @@ export interface IS3Options {
  * @param options music-metadata options
  * @return Tokenizer
  */
-export async function makeTokenizer(s3: S3Client, objRequest: GetObjectRequest, options?: IS3Options): Promise<strtok3.ITokenizer> {
+export async function makeTokenizer(s3: S3Client, objRequest: GetObjectRequest, options?: IS3Options): Promise<ITokenizer> {
   const s3request = new S3Request(s3, objRequest);
   if (options && options.disableChunked) {
     const info = await s3request.getRangedRequest([0, 0]);
     const contentRange = parseContentRange(info.ContentRange);
     const output = await s3.send(new GetObjectCommand(objRequest));
-    return strtok3.fromStream(output.Body as Readable, {
-      mimeType: info.ContentType,
-      size: contentRange.instanceLength
+    return fromStream(output.Body as Readable, {
+      fileInfo: {
+        mimeType: info.ContentType,
+        size: contentRange.instanceLength
+      }
     });
   } else {
     return tokenizer(s3request, {

lib/s3-request.ts

@@ -1,5 +1,5 @@
-import { IRangeRequestClient, IRangeRequestResponse, parseContentRange } from '@tokenizer/range';
-import { S3Client, GetObjectRequest, GetObjectCommand } from '@aws-sdk/client-s3';
+import { type IRangeRequestClient, type IRangeRequestResponse, parseContentRange } from '@tokenizer/range';
+import { S3Client, GetObjectRequest, GetObjectCommand, type GetObjectCommandOutput } from '@aws-sdk/client-s3';
 import { Readable } from 'stream';
 
 /**
@@ -10,33 +10,48 @@ export class S3Request implements IRangeRequestClient {
   constructor(private s3: S3Client, private objRequest: GetObjectRequest) {
   }
 
-  public buildArrayBuffer(body): () => Promise<Buffer> {
-    return async () => {
-      const buffer = [];
-      if (body instanceof Readable) {
-        for await (const chunk of body) {
-          buffer.push(chunk);
-        }
-        return Buffer.concat(buffer);
-      } else {
-        throw new Error('Runtime not supported');
+  /**
+   * Concatenate given array of Uint8Arrays
+   * @param arrays Array of Uint8Arrays
+   */
+  private static mergeUint8Arrays(...arrays: Uint8Array[]): Uint8Array {
+    const totalSize = arrays.reduce((acc, e) => acc + e.length, 0);
+    const merged = new Uint8Array(totalSize);
+
+    arrays.forEach((array, i, arrays) => {
+      const offset = arrays.slice(0, i).reduce((acc, e) => acc + e.length, 0);
+      merged.set(array, offset);
+    });
+
+    return merged;
+  }
+
+  public async buildArrayBuffer(response: GetObjectCommandOutput): Promise<Uint8Array> {
+    const buffers: Uint8Array[] = [];
+    if (response.Body instanceof Readable) {
+      for await (const chunk of response.Body) {
+        buffers.push(chunk);
       }
+      return S3Request.mergeUint8Arrays(...buffers);
+    } else {
+      throw new Error('body expected to be an instance of Readable ');
     }
   }
 
   public async getResponse(method, range: number[]): Promise<IRangeRequestResponse> {
 
-    const response = await this.getRangedRequest(range);
+    const response: GetObjectCommandOutput = await this.getRangedRequest(range);
 
     const { Body: body, ContentType: mimeType } = response;
 
     const contentRange = parseContentRange(response.ContentRange);
 
     return {
       size: contentRange?.instanceLength,
-      mimeType,
+      mimeType: response.ContentType,
       contentRange,
-      arrayBuffer: this.buildArrayBuffer(body),
+      acceptPartialRequests: true,
+      arrayBuffer: () => this.buildArrayBuffer(response),
     };
   }
 

package.json

@@ -2,15 +2,16 @@
   "name": "@tokenizer/s3",
   "version": "0.2.3",
   "description": "Amazon S3 tokenizer",
-  "main": "lib/index.js",
+  "type": "module",
+  "exports": "./lib/index.js",
   "types": "lib/index.d.ts",
   "files": [
     "lib/**/*.js",
     "lib/**/*.d.ts"
   ],
   "scripts": {
     "clean": "del-cli lib/**/*.js lib/**/*.js.map lib/**/*.d.ts test/**/*.js test/**/*.js.map coverage",
-    "mocha": "mocha --require ts-node/register --require source-map-support/register --full-trace test/test.ts",
+    "mocha": "mocha",
     "test": "npm run lint && npm run mocha",
     "compile-lib": "tsc -p lib/tsconfig.json",
     "compile-test": "tsc -p test",
@@ -51,24 +52,27 @@
     "url": "https://github.com/Borewit/tokenizer-s3/issues"
   },
   "dependencies": {
-    "@tokenizer/range": "^0.5.1",
-    "strtok3": "6.3.0"
+    "@aws-sdk/credential-providers": "^3.617.0",
+    "@tokenizer/range": "^0.8.0",
+    "strtok3": "^8.0.1"
   },
   "devDependencies": {
-    "@aws-sdk/client-s3": "^3.223.0",
+    "@aws-sdk/client-s3": "^3.617.0",
     "@tokenizer/token": "^0.3.0",
-    "@types/mocha": "^8.0.0",
-    "@types/node": "^18.11.9",
+    "@types/mocha": "^10.0.7",
+    "@types/node": "^20.14.12",
     "@typescript-eslint/eslint-plugin": "^4.0.0",
     "@typescript-eslint/parser": "^3.0.0",
-    "chai": "^4.3.7",
+    "chai": "^5.1.1",
     "del-cli": "^5.0.0",
     "eslint": "^7.0.0",
-    "mocha": "^9.2.2",
-    "ts-node": "^10.9.1",
-    "typescript": "^4.9.3"
+    "file-type": "^19.3.0",
+    "mocha": "^10.7.0",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.5.4"
   },
   "peerDependencies": {
     "@aws-sdk/client-s3": "^3.223.0"
-  }
+  },
+  "packageManager": "yarn@1.22.22+sha512.a6b2f7906b721bba3d67d4aff083df04dad64c399707841b7acf00f6b133b7ac24255f2652fa22ae3534329dc6180534e98d17432037ff6fd140556e2bb3137e"
 }

test/test.ts

@@ -1,11 +1,15 @@
+import { fromEnv } from '@aws-sdk/credential-providers';
 import { S3Client } from '@aws-sdk/client-s3';
 import { assert } from 'chai';
-import { makeTokenizer } from '../lib';
+import { makeTokenizer } from '../lib/index.js';
 
 describe('S3 Tokenizer', function() {
 
   this.timeout(20000);
-  const s3 = new S3Client({});
+  const s3 = new S3Client({
+    region: 'eu-west-2',
+    credentials: fromEnv(),
+  });
 
   describe('initialize tokenizer.fileInfo', () => {
 

tsconfig.json

@@ -1,9 +1,9 @@
 {
   "compilerOptions": {
     "inlineSources": false,
-    "module": "commonjs",
-    "moduleResolution": "node",
-    "target": "ES2017",
+    "module": "Node16",
+    "moduleResolution": "Node16",
+    "target": "ES2020",
     "sourceMap": true
   }
 }