feat!: upgrade AWS SDK to v3

Dockerfile

@@ -1,14 +1,14 @@
-# Needed for DynamoDB Local, which is used for unit tests
-FROM openjdk:15-alpine3.11
+# Java JDK is needed for DynamoDB Local, which is used for unit tests
+FROM amazoncorretto:18-alpine3.15
 
 # And our own stuff goes here
 WORKDIR /usr/app
 COPY . ./
 RUN apk add --update \
     yarn \
-    python \
-    python-dev \
+    python3 \
+    python3-dev \
     py-pip \
     build-base \
     nodejs \
-    npm
+    npm

README.md

@@ -132,7 +132,7 @@ Now you can write partition-aware, type safe queries with abandon:
 
 ```TypeScript
 import { Beyonce } from "@ginger.io/beyonce"
-import { DynamoDB } from "aws-sdk"
+import { DynamoDB } from "@aws-sdk/client-dynamodb"
 import { LibraryTable } from "generated/models"
 
 const beyonce = new Beyonce(LibraryTable, dynamo)

jest-dynamodb-config.js

@@ -12,8 +12,10 @@ module.exports = async () => {
     port,
     clientConfig: {
       endpoint,
-      sslEnabled: false,
-      region: "local",
+      credentials: {
+        accessKeyId: "foo",
+        secretAccessKey: "baz",
+      },
     },
     options: ["-inMemory"],
     tables: [],

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ginger.io/beyonce",
-  "version": "0.0.66",
+  "version": "0.0.67",
   "description": "Type-safe DynamoDB query builder for TypeScript. Designed with single-table architecture in mind.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -12,8 +12,9 @@
     "./src"
   ],
   "dependencies": {
+    "@aws-sdk/client-dynamodb": "3.113.0",
+    "@aws-sdk/lib-dynamodb": "3.113.0",
     "@ginger.io/jay-z": "0.0.14",
-    "aws-sdk": "^2.853.0",
     "aws-xray-sdk": "^3.2.0",
     "fast-json-stable-stringify": "^2.1.0",
     "js-yaml": "^3.13.1",
@@ -22,7 +23,7 @@
     "yargs": "^15.3.1"
   },
   "devDependencies": {
-    "@shelf/jest-dynamodb": "^1.7.0",
+    "@shelf/jest-dynamodb": "3.3.0",
     "@types/jest": "^25.1.4",
     "@types/js-yaml": "^3.12.2",
     "@types/libsodium-wrappers": "^0.7.8",

src/main/dom.d.ts

@@ -0,0 +1,10 @@
+export {}
+
+// @aws-sdk/client-dynamodb version 3.150.0 causes the build to fail if targeting a Nodejs environment instead of dom,
+// since it expects several types that are available only in the browser.
+// See https://stackoverflow.com/a/69581652/17092655.
+declare global {
+  type ReadableStream = unknown
+  type Blob = unknown
+  type File = unknown
+}

src/main/dynamo/Beyonce.ts

@@ -1,5 +1,6 @@
 import { JayZ } from "@ginger.io/jay-z"
-import { DynamoDB } from "aws-sdk"
+import { DynamoDB } from "@aws-sdk/client-dynamodb"
+import { DynamoDBDocumentClient } from "@aws-sdk/lib-dynamodb"
 import { captureAWSClient } from "aws-xray-sdk"
 import { UpdateItemExpressionBuilder } from "./expressions/UpdateItemExpressionBuilder"
 import { groupModelsByType } from "./groupModelsByType"
@@ -16,7 +17,7 @@ import { ParallelScanConfig, ScanBuilder } from "./ScanBuilder"
 import { Table } from "./Table"
 import { ExtractKeyType, GroupedModels, TaggedModel } from "./types"
 import { updateItemProxy } from "./updateItemProxy"
-import { decryptOrPassThroughItem, encryptOrPassThroughItem, MaybeEncryptedItem } from "./util"
+import { decryptOrPassThroughItem, encryptOrPassThroughItem, formatDynamoDBItem, MaybeEncryptedItem } from "./util"
 
 export interface Options {
   jayz?: JayZ
@@ -41,12 +42,14 @@ export interface ScanOptions {
  * does auto mapping between JSON <=> DynamoDB Items
  */
 export class Beyonce {
-  private client: DynamoDB.DocumentClient
+  private client: DynamoDBDocumentClient
   private jayz?: JayZ
   private consistentReads: boolean
 
   constructor(private table: Table<string, string>, dynamo: DynamoDB, options: Options = {}) {
-    this.client = new DynamoDB.DocumentClient({ service: dynamo })
+    this.client = DynamoDBDocumentClient.from(dynamo, {
+      marshallOptions: { removeUndefinedValues: true }
+    })
     if (options.xRayTracingEnabled) {
       // hack per: https://github.com/aws/aws-xray-sdk-node/issues/23#issuecomment-509745488
       captureAWSClient((this.client as any).service)
@@ -139,7 +142,8 @@ export class Beyonce {
     const items: ExtractKeyType<T>[] = []
     const unprocessedKeys: T[] = []
     results.forEach((result) => {
-      items.push(...result.items)
+      const formattedItems = result.items.map(item => item ? formatDynamoDBItem(item) : item)
+      items.push(...formattedItems)
       unprocessedKeys.push(...result.unprocessedKeys)
     })
 

src/main/dynamo/QueryBuilder.ts

@@ -1,6 +1,5 @@
 import { JayZ } from "@ginger.io/jay-z"
-import { DynamoDB } from "aws-sdk"
-import { DocumentClient } from "aws-sdk/clients/dynamodb"
+import { DynamoDBDocumentClient, QueryCommand, QueryCommandInput } from "@aws-sdk/lib-dynamodb"
 import { ready } from "libsodium-wrappers"
 import { QueryExpressionBuilder } from "./expressions/QueryExpressionBuilder"
 import { groupModelsByType } from "./groupModelsByType"
@@ -12,15 +11,15 @@ import { Table } from "./Table"
 import { GroupedModels, TaggedModel } from "./types"
 
 interface TableQueryConfig<T extends TaggedModel> {
-  db: DynamoDB.DocumentClient
+  db: DynamoDBDocumentClient
   table: Table
   key: PartitionKey<T> | PartitionKeyAndSortKeyPrefix<T>
   jayz?: JayZ
   consistentRead?: boolean
 }
 
 interface GSIQueryConfig<T extends TaggedModel> {
-  db: DynamoDB.DocumentClient
+  db: DynamoDBDocumentClient
   table: Table
   gsiName: string
   gsiKey: PartitionKey<T>
@@ -44,14 +43,14 @@ export class QueryBuilder<T extends TaggedModel> extends QueryExpressionBuilder<
 
   async exec(): Promise<GroupedModels<T>> {
     const query = this.createQueryInput({ lastEvaluatedKey: undefined })
-    const iterator = pagedIterator<DocumentClient.QueryInput, T>(
+    const iterator = pagedIterator<QueryCommandInput, T>(
       { lastEvaluatedKey: undefined },
       ({ lastEvaluatedKey, pageSize }) => ({
         ...query,
         ExclusiveStartKey: lastEvaluatedKey,
         Limit: pageSize
       }),
-      (query) => this.config.db.query(query).promise(),
+      (query) => this.config.db.send(new QueryCommand(query)),
       this.config.jayz
     )
 
@@ -61,14 +60,14 @@ export class QueryBuilder<T extends TaggedModel> extends QueryExpressionBuilder<
   async *iterator(options: IteratorOptions = {}): PaginatedIteratorResults<T> {
     const iteratorOptions = toInternalIteratorOptions(options)
     const query = this.createQueryInput(iteratorOptions)
-    const iterator = pagedIterator<DocumentClient.QueryInput, T>(
+    const iterator = pagedIterator<QueryCommandInput, T>(
       iteratorOptions,
       ({ lastEvaluatedKey, pageSize }) => ({
         ...query,
         ExclusiveStartKey: lastEvaluatedKey,
         Limit: pageSize
       }),
-      (query) => this.config.db.query(query).promise(),
+      (query) => this.config.db.send(new QueryCommand(query)),
       this.config.jayz
     )
 
@@ -88,7 +87,7 @@ export class QueryBuilder<T extends TaggedModel> extends QueryExpressionBuilder<
     }
   }
 
-  private createQueryInput(options: InternalIteratorOptions): DynamoDB.DocumentClient.QueryInput {
+  private createQueryInput(options: InternalIteratorOptions): QueryCommandInput {
     if (isTableQuery(this.config)) {
       const { table, consistentRead } = this.config
       const keyCondition = this.buildKeyConditionForTable(this.config)

src/main/dynamo/ScanBuilder.ts

@@ -1,6 +1,5 @@
 import { JayZ } from "@ginger.io/jay-z"
-import { DynamoDB } from "aws-sdk"
-import { DocumentClient } from "aws-sdk/clients/dynamodb"
+import { DynamoDBDocumentClient, ScanCommand, ScanCommandInput } from "@aws-sdk/lib-dynamodb"
 import { ready } from "libsodium-wrappers"
 import { QueryExpressionBuilder } from "./expressions/QueryExpressionBuilder"
 import { groupModelsByType } from "./groupModelsByType"
@@ -11,7 +10,7 @@ import { Table } from "./Table"
 import { GroupedModels, TaggedModel } from "./types"
 
 interface ScanConfig<T extends TaggedModel> {
-  db: DynamoDB.DocumentClient
+  db: DynamoDBDocumentClient
   table: Table
   jayz?: JayZ
   consistentRead?: boolean
@@ -33,14 +32,14 @@ export class ScanBuilder<T extends TaggedModel> extends QueryExpressionBuilder<T
 
   async exec(): Promise<GroupedModels<T>> {
     const scanInput = this.createScanInput()
-    const iterator = pagedIterator<DocumentClient.ScanInput, T>(
+    const iterator = pagedIterator<ScanCommandInput, T>(
       { lastEvaluatedKey: undefined },
       ({ lastEvaluatedKey, pageSize }) => ({
         ...scanInput,
         ExclusiveStartKey: lastEvaluatedKey,
         Limit: pageSize
       }),
-      (input) => this.config.db.scan(input).promise(),
+      (input) => this.config.db.send(new ScanCommand(input)),
       this.config.jayz
     )
 
@@ -50,14 +49,14 @@ export class ScanBuilder<T extends TaggedModel> extends QueryExpressionBuilder<T
   async *iterator(options: IteratorOptions = {}): PaginatedIteratorResults<T> {
     const iteratorOptions = toInternalIteratorOptions(options)
     const scanInput = this.createScanInput(iteratorOptions)
-    const iterator = pagedIterator<DocumentClient.ScanInput, T>(
+    const iterator = pagedIterator<ScanCommandInput, T>(
       iteratorOptions,
       ({ lastEvaluatedKey, pageSize }) => ({
         ...scanInput,
         ExclusiveStartKey: lastEvaluatedKey,
         Limit: pageSize
       }),
-      (input) => this.config.db.scan(input).promise(),
+      (input) => this.config.db.send(new ScanCommand(input)),
       this.config.jayz
     )
 

src/main/dynamo/Table.ts

@@ -1,4 +1,4 @@
-import { DynamoDB } from "aws-sdk"
+import { AttributeDefinition, BillingMode, CreateTableInput } from "@aws-sdk/client-dynamodb"
 import { GSI, GSIBuilder } from "./GSI"
 import { Model, PartitionKeyBuilder } from "./Model"
 import { Partition } from "./Partition"
@@ -53,15 +53,15 @@ export class Table<PK extends string = string, SK extends string = string> {
     return this.modelTags
   }
 
-  asCreateTableInput(billingMode: DynamoDB.Types.BillingMode): DynamoDB.Types.CreateTableInput {
+  asCreateTableInput(billingMode: BillingMode): CreateTableInput {
     const attributeSet = new Set([
       this.partitionKeyName,
       this.sortKeyName,
       "model",
       ...this.gsis.flatMap((_) => [_.partitionKeyName, _.sortKeyName])
     ])
 
-    const attributeDefinitions: DynamoDB.Types.AttributeDefinitions = Array.from(attributeSet).map((attr) => ({
+    const attributeDefinitions: AttributeDefinition[] = Array.from(attributeSet).map((attr) => ({
       AttributeName: attr,
       AttributeType: "S"
     }))

src/main/dynamo/UnprocessedKeyCollector.ts

@@ -1,7 +1,6 @@
-import { DynamoDB } from "aws-sdk"
 import { PartitionAndSortKey } from "./keys"
 import { Table } from "./Table"
-import { TaggedModel } from "./types"
+import { Key, TaggedModel } from "./types"
 
 /** Collects a set of unprocessed DynamoDB keys returned from an operation (e.g. batchGet) and maps the "raw"
  *  key Dynamo returns back to the Beyonce key (PartitionAndSortKey class instance)
@@ -18,7 +17,7 @@ export class UnprocessedKeyCollector<T extends PartitionAndSortKey<TaggedModel>>
     inputKeys.forEach((key) => (this.dynamoKeyToBeyonceKey[`${key.partitionKey}-${key.sortKey}`] = key))
   }
 
-  add(unprocessedKey: DynamoDB.DocumentClient.Key): void {
+  add(unprocessedKey: Key): void {
     const { partitionKeyName, sortKeyName } = this.table
     const beyonceKey = this.dynamoKeyToBeyonceKey[`${unprocessedKey[partitionKeyName]}-${unprocessedKey[sortKeyName]}`]
     if (beyonceKey) {

src/main/dynamo/expressions/DynamoDBExpression.ts

@@ -1,7 +1,5 @@
-import { DynamoDB } from "aws-sdk"
-
 export type DynamoDBExpression = {
-  expression: DynamoDB.DocumentClient.ConditionExpression
-  attributeNames: DynamoDB.DocumentClient.ExpressionAttributeNameMap
-  attributeValues: DynamoDB.DocumentClient.ExpressionAttributeValueMap
+  expression: string
+  attributeNames: Record<string, string>
+  attributeValues: Record<string, string>
 }

src/main/dynamo/iterators/pagedIterator.ts

@@ -1,21 +1,15 @@
 import { JayZ } from "@ginger.io/jay-z"
-import { DynamoDB } from "aws-sdk"
-import { DocumentClient } from "aws-sdk/clients/dynamodb"
+import { QueryCommandOutput } from "@aws-sdk/lib-dynamodb"
 import { CompositeError } from "../../CompositeError"
 import { groupModelsByType } from "../groupModelsByType"
-import { GroupedModels, TaggedModel } from "../types"
-import { decryptOrPassThroughItem } from "../util"
+import { GroupedModels, Key, TaggedModel } from "../types"
+import { decryptOrPassThroughItem, formatDynamoDBItem } from "../util"
 import { InternalIteratorOptions } from "./types"
 
-export type RawDynamoDBPage = {
-  LastEvaluatedKey?: DocumentClient.Key
-  Items?: DocumentClient.ItemList
-}
-
 export type PageResults<T extends TaggedModel> = {
   items: T[]
   errors: Error[]
-  lastEvaluatedKey?: DynamoDB.DocumentClient.Key
+  lastEvaluatedKey?: Key
 }
 
 export async function groupAllPages<T extends TaggedModel>(
@@ -27,8 +21,8 @@ export async function groupAllPages<T extends TaggedModel>(
     if (errors.length > 0) {
       throw new CompositeError("Error(s) encountered trying to process interator page", errors)
     }
-
-    results.push(...items)
+    const formattedItems = items.map(item => formatDynamoDBItem<T>(item))
+    results.push(...formattedItems)
   }
 
   return groupModelsByType(results, modelTags)
@@ -37,7 +31,7 @@ export async function groupAllPages<T extends TaggedModel>(
 export async function* pagedIterator<T, U extends TaggedModel>(
   options: InternalIteratorOptions,
   buildOperation: (opts: InternalIteratorOptions) => T,
-  executeOperation: (op: T) => Promise<RawDynamoDBPage>,
+  executeOperation: (op: T) => Promise<QueryCommandOutput>,
   jayz?: JayZ
 ): AsyncGenerator<PageResults<U>, PageResults<U>> {
   let pendingOperation: T | undefined = buildOperation(options)
@@ -47,7 +41,7 @@ export async function* pagedIterator<T, U extends TaggedModel>(
     const items: U[] = []
     const errors: Error[] = []
     try {
-      const response: DynamoDB.DocumentClient.QueryOutput = await executeOperation(pendingOperation)
+      const response: QueryCommandOutput = await executeOperation(pendingOperation)
 
       if (response.LastEvaluatedKey !== undefined) {
         lastEvaluatedKey = response.LastEvaluatedKey

src/main/dynamo/iterators/types.ts

@@ -1,5 +1,4 @@
-import { DocumentClient } from "aws-sdk/clients/dynamodb"
-import { GroupedModels, TaggedModel } from "../types"
+import { GroupedModels, Key, TaggedModel } from "../types"
 
 export type Cursor = string
 
@@ -11,7 +10,7 @@ export interface IteratorOptions {
 
 /** Iterator options intended for internal use (within this library) */
 export interface InternalIteratorOptions {
-  lastEvaluatedKey: DocumentClient.Key | undefined
+  lastEvaluatedKey: Key | undefined
   pageSize?: number
 }
 

src/main/dynamo/iterators/util.ts

@@ -1,8 +1,9 @@
-import { Cursor } from "aws-sdk/clients/cloudsearchdomain"
-import { DocumentClient } from "aws-sdk/clients/dynamodb"
 import { base64_variants, from_base64, to_base64 } from "libsodium-wrappers"
 import { TextDecoder } from "util"
-import { InternalIteratorOptions, IteratorOptions } from "./types"
+import { InternalIteratorOptions, IteratorOptions} from "./types"
+import { Key } from "main/dynamo/types"
+
+type Cursor = string
 
 export function toInternalIteratorOptions(options: IteratorOptions): InternalIteratorOptions {
   return {
@@ -11,13 +12,13 @@ export function toInternalIteratorOptions(options: IteratorOptions): InternalIte
   }
 }
 
-export function maybeSerializeCursor(key?: DocumentClient.Key): string | undefined {
+export function maybeSerializeCursor(key?: Key): string | undefined {
   if (key) {
     return to_base64(JSON.stringify(key), base64_variants.ORIGINAL_NO_PADDING)
   }
 }
 
-export function maybeDeserializeCursor(cursor?: Cursor): DocumentClient.Key | undefined {
+export function maybeDeserializeCursor(cursor?: Cursor): Key | undefined {
   if (cursor) {
     const json = new TextDecoder().decode(from_base64(cursor, base64_variants.ORIGINAL_NO_PADDING))
     return JSON.parse(json)