chore(core): create submodule exports in core (#6079)

* chore(core): create submodule exports in core

* chore(core): organize submodules

* chore(core): submodules linting

* chore(core): submodule readmes

* chore(core): update readme about Metro package exports support

* chore(core): add compatibility redirect files for core submodules

* chore(core): add files entry in pkg.json

* chore(core): add files pkg.json enforcement

* chore(core): import submodules in core root index

README.md

@@ -110,6 +110,11 @@ import "react-native-url-polyfill/auto";
 import { DynamoDB } from "@aws-sdk/client-dynamodb";
 ```
 
+Specifically Metro bundler used by react-native, enable Package Exports Support:
+
+- https://metrobundler.dev/docs/package-exports/
+- https://reactnative.dev/blog/2023/06/21/package-exports-support
+
 ## New features
 
 ### Modularized packages

packages/cloudfront-signer/package.json

@@ -24,6 +24,9 @@
     "@smithy/url-parser": "^3.0.0",
     "tslib": "^2.6.2"
   },
+  "files": [
+    "dist-*/**"
+  ],
   "devDependencies": {
     "@tsconfig/recommended": "1.0.1",
     "concurrently": "7.0.0",

packages/core/README.md

@@ -1,5 +1,39 @@
-# @aws-sdk/core
+# `@aws-sdk/core`
 
 This package provides common or core functionality to the AWS SDK for JavaScript (v3).
 
 You do not need to explicitly install this package, since it will be transitively installed by AWS SDK clients.
+
+## `@aws-sdk/core` submodules
+
+Core submodules are organized for distribution via the `package.json` `exports` field.
+
+`exports` is supported by default by the latest Node.js, webpack, and esbuild. For react-native, it can be
+enabled via instructions found at [reactnative.dev/blog](https://reactnative.dev/blog/2023/06/21/package-exports-support).
+
+Think of `@aws-sdk/core` as a mono-package within the monorepo.
+It preserves the benefits of modularization, for example to optimize Node.js initialization speed,
+while making it easier to have a consistent version of core dependencies, reducing package sprawl when
+installing an SDK client.
+
+### Guide for submodules
+
+- Each `index.ts` file corresponding to the pattern `./src/submodules/<MODULE_NAME>/index.ts` will be
+  published as a separate `dist-cjs` bundled submodule index using the `Inliner.js` build script.
+- create a folder as `./src/submodules/<SUBMODULE>` including an `index.ts` file and a `README.md` file.
+  - The linter will throw an error on missing submodule metadata in `package.json` and the various `tsconfig.json` files, but it will automatically fix them if possible.
+- a submodule is equivalent to a standalone `@aws-sdk/<pkg>` package in that importing it in Node.js will resolve a separate bundle.
+- submodules may not relatively import files from other submodules. Instead, directly use the `@scope/pkg/submodule` name as the import.
+  - The linter will check for this and throw an error.
+- To the extent possible, correctly declaring submodule metadata is validated by the linter in `@aws-sdk/core`.
+  The linter runs during `yarn build` and also as `yarn lint`.
+
+### When should I create an `@aws-sdk/core/submodule` vs. `@aws-sdk/new-package`?
+
+Keep in mind that the core package is installed by all AWS SDK clients.
+
+If the component functionality is upstream of multiple clients, it is
+a good candidate for a core submodule. For example, XML serialization.
+
+If the component's functionality is downstream of a client, for example S3 pre-signing,
+it should be a standalone package with potentially a peer or runtime dependency on an AWS SDK client.

packages/core/client.js

@@ -0,0 +1,5 @@
+/**
+ * Do not edit:
+ * This is a compatibility redirect for contexts that do not understand package.json exports field.
+ */
+module.exports = require("./dist-cjs/submodules/client/index.js");

packages/core/httpAuthSchemes.js

@@ -0,0 +1,5 @@
+/**
+ * Do not edit:
+ * This is a compatibility redirect for contexts that do not understand package.json exports field.
+ */
+module.exports = require("./dist-cjs/submodules/httpAuthSchemes/index.js");

packages/core/package.json

@@ -3,7 +3,7 @@
   "version": "3.575.0",
   "description": "Core functions & classes shared by multiple AWS SDK clients",
   "scripts": {
-    "build": "concurrently 'yarn:build:cjs' 'yarn:build:es' 'yarn:build:types'",
+    "build": "yarn lint && concurrently 'yarn:build:cjs' 'yarn:build:es' 'yarn:build:types'",
     "build:cjs": "node ../../scripts/compilation/inline core",
     "build:es": "tsc -p tsconfig.es.json",
     "build:include:deps": "lerna run --scope $npm_package_name --include-dependencies build",
@@ -18,6 +18,43 @@
   "main": "./dist-cjs/index.js",
   "module": "./dist-es/index.js",
   "types": "./dist-types/index.d.ts",
+  "exports": {
+    ".": {
+      "node": "./dist-cjs/index.js",
+      "import": "./dist-es/index.js",
+      "require": "./dist-cjs/index.js",
+      "types": "./dist-types/index.d.ts"
+    },
+    "./package.json": {
+      "node": "./package.json",
+      "import": "./package.json",
+      "require": "./package.json"
+    },
+    "./client": {
+      "node": "./dist-cjs/submodules/client/index.js",
+      "import": "./dist-es/submodules/client/index.js",
+      "require": "./dist-cjs/submodules/client/index.js",
+      "types": "./dist-types/submodules/client/index.d.ts"
+    },
+    "./httpAuthSchemes": {
+      "node": "./dist-cjs/submodules/httpAuthSchemes/index.js",
+      "import": "./dist-es/submodules/httpAuthSchemes/index.js",
+      "require": "./dist-cjs/submodules/httpAuthSchemes/index.js",
+      "types": "./dist-types/submodules/httpAuthSchemes/index.d.ts"
+    },
+    "./protocols": {
+      "node": "./dist-cjs/submodules/protocols/index.js",
+      "import": "./dist-es/submodules/protocols/index.js",
+      "require": "./dist-cjs/submodules/protocols/index.js",
+      "types": "./dist-types/submodules/protocols/index.d.ts"
+    }
+  },
+  "files": [
+    "dist-*/**",
+    "./client.js",
+    "./httpAuthSchemes.js",
+    "./protocols.js"
+  ],
   "sideEffects": false,
   "author": {
     "name": "AWS SDK for JavaScript Team",

packages/core/protocols.js

@@ -0,0 +1,5 @@
+/**
+ * Do not edit:
+ * This is a compatibility redirect for contexts that do not understand package.json exports field.
+ */
+module.exports = require("./dist-cjs/submodules/protocols/index.js");

packages/core/scripts/lint.js

@@ -1,24 +1,98 @@
 const fs = require("fs");
 const path = require("path");
-const assert = require("assert");
 
 const root = path.join(__dirname, "..");
 
 const pkgJson = require(path.join(root, "package.json"));
-const srcFolders = fs.readdirSync(path.join(root, "src"));
+const tsconfigs = {
+  cjs: require(path.join(root, "tsconfig.cjs.json")),
+  es: require(path.join(root, "tsconfig.es.json")),
+  types: require(path.join(root, "tsconfig.types.json")),
+};
+const submodules = fs.readdirSync(path.join(root, "src", "submodules"));
 
-assert(pkgJson.exports === undefined, "We cannot support package.json exports yet.");
+const errors = [];
 
+for (const submodule of submodules) {
+  const submodulePath = path.join(root, "src", "submodules", submodule);
+  if (fs.existsSync(submodulePath) && fs.lstatSync(submodulePath).isDirectory()) {
+    // package.json metadata.
+    if (!pkgJson.exports[`./${submodule}`]) {
+      errors.push(`${submodule} submodule is missing exports statement in package.json`);
+      pkgJson.exports[`./${submodule}`] = {
+        node: `./dist-cjs/submodules/${submodule}/index.js`,
+        import: `./dist-es/submodules/${submodule}/index.js`,
+        require: `./dist-cjs/submodules/${submodule}/index.js`,
+        types: `./dist-types/submodules/${submodule}/index.d.ts`,
+      };
+      fs.writeFileSync(path.join(root, "package.json"), JSON.stringify(pkgJson, null, 2) + "\n");
+    }
+    if (!pkgJson.files.includes(`./${submodule}.js`)) {
+      pkgJson.files.push(`./${submodule}.js`);
+      errors.push(`package.json files array missing ${submodule}.js compatibility redirect file.`);
+      fs.writeFileSync(path.join(root, "package.json"), JSON.stringify(pkgJson, null, 2) + "\n");
+    }
+    // tsconfig metadata.
+    for (const [kind, tsconfig] of Object.entries(tsconfigs)) {
+      if (!tsconfig.compilerOptions.paths?.[`@aws-sdk/core/${submodule}`]) {
+        errors.push(`${submodule} submodule is missing paths entry in tsconfig.${kind}.json`);
+
+        tsconfig.compilerOptions.paths[`@aws-sdk/core/${submodule}`] = [`./src/submodules/${submodule}/index.ts`];
+        fs.writeFileSync(path.join(root, `tsconfig.${kind}.json`), JSON.stringify(tsconfig, null, 2) + "\n");
+      }
+    }
+    // compatibility redirect file.
+    const compatibilityRedirectFile = path.join(root, `${submodule}.js`);
+    if (!fs.existsSync(compatibilityRedirectFile)) {
+      errors.push(`${submodule} is missing compatibility redirect file in the package root folder.`);
+      fs.writeFileSync(
+        compatibilityRedirectFile,
+        `
 /**
- * We probably can't enable package.json exports until
- * dropping support for Node.js 14.x and TypeScript 4.6.
+ * Do not edit:
+ * This is a compatibility redirect for contexts that do not understand package.json exports field.
  */
-process.exit(0);
-
-for (const srcFolder of srcFolders) {
-  if (fs.lstatSync(path.join(root, "src", srcFolder)).isDirectory()) {
-    if (!pkgJson.exports["./" + srcFolder]) {
-      throw new Error(`${srcFolder} is missing exports statement in package.json`);
+module.exports = require("./dist-cjs/submodules/${submodule}/index.js");
+`
+      );
     }
   }
 }
+
+/**
+ * Check for cross-submodule relative imports.
+ */
+
+const walk = require("../../../scripts/utils/walk");
+
+(async () => {
+  for await (const item of walk(path.join(root, "src", "submodules"))) {
+    // depth within the submodule where 1 is at the root of the submodule.
+    const depth = item.split("core/src/submodules/")[1].split("/").length - 1;
+    const sourceCode = fs.readFileSync(item, "utf-8");
+
+    const relativeImports = [];
+    relativeImports.push(
+      ...new Set(
+        [...(sourceCode.toString().match(/(from |import\()"(.*?)";/g) || [])]
+          .map((_) => _.replace(/from "/g, "").replace(/";$/, ""))
+          .filter((_) => _.startsWith("."))
+      )
+    );
+
+    for (const i of relativeImports) {
+      const relativeImportDepth = i.split("..").length - 1;
+      if (relativeImportDepth >= depth) {
+        errors.push(
+          `relative import ${i} in ${item
+            .split("packages/")
+            .pop()} crosses submodule boundaries. Use @scope/package/submodule import instead.`
+        );
+      }
+    }
+  }
+})().then(() => {
+  if (errors.length) {
+    throw new Error(errors.join("\n"));
+  }
+});

packages/core/src/index.ts

@@ -1,3 +1,24 @@
-export * from "./client/index";
-export * from "./httpAuthSchemes/index";
-export * from "./protocols/index";
+/**
+ * Submodules annotated with "Legacy" are from prior to the submodule system.
+ * They are exported from the package's root index to preserve backwards compatibility.
+ *
+ * New development should go in a proper submodule and not be exported from the root index.
+ */
+
+/**
+ * Legacy submodule.
+ */
+export * from "./submodules/client/index";
+/**
+ * Legacy submodule.
+ */
+export * from "./submodules/httpAuthSchemes/index";
+/**
+ * Legacy submodule.
+ */
+export * from "./submodules/protocols/index";
+
+/**
+ * Warning: do not export any additional submodules from the root of this package. See readme.md for
+ * guide on developing submodules.
+ */

packages/core/src/submodules/client/README.md

@@ -0,0 +1,3 @@
+# @aws-sdk/core/client
+
+This is a legacy submodule that is also exported at the root index.

packages/core/src/submodules/httpAuthSchemes/README.md

@@ -0,0 +1,3 @@
+# @aws-sdk/core/httpAuthSchemes
+
+This is a legacy submodule that is also exported at the root index.

packages/core/src/submodules/protocols/README.md

@@ -0,0 +1,3 @@
+# @aws-sdk/core/protocols
+
+This is a legacy submodule that is also exported at the root index.

packages/core/tsconfig.cjs.json

@@ -2,7 +2,12 @@
   "compilerOptions": {
     "rootDir": "./src",
     "outDir": "./dist-cjs",
-    "baseUrl": "."
+    "baseUrl": ".",
+    "paths": {
+      "@aws-sdk/core/client": ["./src/submodules/client/index.ts"],
+      "@aws-sdk/core/httpAuthSchemes": ["./src/submodules/httpAuthSchemes/index.ts"],
+      "@aws-sdk/core/protocols": ["./src/submodules/protocols/index.ts"]
+    }
   },
   "extends": "../../tsconfig.cjs.json",
   "include": ["src/"]

packages/core/tsconfig.es.json

@@ -2,7 +2,12 @@
   "compilerOptions": {
     "rootDir": "./src",
     "outDir": "./dist-es",
-    "baseUrl": "."
+    "baseUrl": ".",
+    "paths": {
+      "@aws-sdk/core/client": ["./src/submodules/client/index.ts"],
+      "@aws-sdk/core/httpAuthSchemes": ["./src/submodules/httpAuthSchemes/index.ts"],
+      "@aws-sdk/core/protocols": ["./src/submodules/protocols/index.ts"]
+    }
   },
   "extends": "../../tsconfig.es.json",
   "include": ["src/"]

packages/core/tsconfig.types.json

@@ -2,7 +2,12 @@
   "compilerOptions": {
     "baseUrl": ".",
     "declarationDir": "dist-types",
-    "rootDir": "src"
+    "rootDir": "src",
+    "paths": {
+      "@aws-sdk/core/client": ["./src/submodules/client/index.ts"],
+      "@aws-sdk/core/httpAuthSchemes": ["./src/submodules/httpAuthSchemes/index.ts"],
+      "@aws-sdk/core/protocols": ["./src/submodules/protocols/index.ts"]
+    }
   },
   "extends": "../../tsconfig.types.json",
   "include": ["src/"]

scripts/compilation/Inliner.js

@@ -19,6 +19,7 @@ module.exports = class Inliner {
     this.isPackage = fs.existsSync(path.join(root, "packages", pkg));
     this.isLib = fs.existsSync(path.join(root, "lib", pkg));
     this.isClient = !this.isPackage && !this.isLib;
+    this.isCore = pkg === "core";
     this.subfolder = this.isPackage ? "packages" : this.isLib ? "lib" : "clients";
 
     this.packageDirectory = path.join(root, this.subfolder, pkg);
@@ -149,9 +150,9 @@ module.exports = class Inliner {
       (variant) => "*/" + path.basename(variant).replace(/.js$/, "")
     );
 
-    await esbuild.build({
+    const buildOptions = {
       platform: this.platform,
-      target: ["node14"],
+      target: ["node16"],
       bundle: true,
       format: "cjs",
       mainFields: ["main"],
@@ -164,7 +165,27 @@ module.exports = class Inliner {
       keepNames: true,
       packages: "external",
       external: ["@smithy/*", "@aws-sdk/*", "node_modules/*", ...this.variantExternalsForEsBuild],
-    });
+    };
+
+    if (!this.isCore) {
+      await esbuild.build(buildOptions);
+    }
+
+    if (this.isCore) {
+      const submodules = fs.readdirSync(path.join(root, this.subfolder, this.package, "src", "submodules"));
+      for (const submodule of submodules) {
+        fs.rmSync(path.join(path.join(root, this.subfolder, this.package, "dist-cjs", "submodules", submodule)), {
+          recursive: true,
+          force: true,
+        });
+        await esbuild.build({
+          ...buildOptions,
+          entryPoints: [path.join(root, this.subfolder, this.package, "src", "submodules", submodule, "index.ts")],
+          outfile: path.join(root, this.subfolder, this.package, "dist-cjs", "submodules", submodule, "index.js"),
+        });
+      }
+    }
+
     return this;
   }
 
@@ -173,7 +194,7 @@ module.exports = class Inliner {
    * These now become re-exports of the index to preserve deep-import behavior.
    */
   async rewriteStubs() {
-    if (this.bailout) {
+    if (this.bailout || this.isCore) {
       return this;
     }
 

scripts/runtime-dependency-version-check/package-json-enforcement.js

@@ -55,6 +55,10 @@ module.exports = function (pkgJsonFilePath, overwrite = false) {
     errors.push(`browser and react-native fields are different in ${pkgJson.name}`);
   }
 
+  if (!pkgJson.files) {
+    errors.push(`no files entry in ${pkgJson.name}`);
+  }
+
   if (typeof pkgJson.browser === "object" && typeof pkgJson["react-native"] === "object") {
     const browserCanonical = Object.entries(pkgJson.browser).reduce((acc, [k, v]) => {
       if (!k.includes("dist-cjs/") || typeof v === "boolean") {