Infra Pipeline: Modular Environment Setups

.eslintrc.js

@@ -2,6 +2,7 @@ module.exports = {
   root: true,
   ignorePatterns: [
     "**/w3/**/*.*",
+    "**/infra-modules/**/*.*",
     "**/build/**/*.*",
     "**/__tests__/**/*.*",
     "**/node_modules/**/*.*",

.github/workflows/js-ci.yaml

@@ -191,7 +191,7 @@ jobs:
 
 # NOTE: Mac & Windows CI is not currently possible, since GitHub Actions does not
 # use "Docker Enterprise". Docker Enterprise is required when running linux based
-# containers, which is what we run within our toolchain for test-envs and build-envs.
+# containers, which is what we run within our toolchain for infra and build-images.
 # In order to enable this, we can move to Circle CI, which supports Docker Enterprise.
 
 #  JS-CI-Mac:

.github/workflows/rs-ci.yaml

@@ -95,5 +95,5 @@ jobs:
 
 # NOTE: Mac & Windows CI is not currently possible, since GitHub Actions does not
 # use "Docker Enterprise". Docker Enterprise is required when running linux based
-# containers, which is what we run within our toolchain for test-envs and build-envs.
+# containers, which is what we run within our toolchain for infra and build-images.
 # In order to enable this, we can move to Circle CI, which supports Docker Enterprise.

package.json

@@ -10,7 +10,7 @@
   "workspaces": {
     "packages": [
       "./packages/apis/**",
-      "./packages/cli/**",
+      "./packages/cli",
       "./packages/core-interfaces/**",
       "./packages/js/**",
       "./packages/manifest-schemas/**",

packages/cli/copyfiles.ts

@@ -0,0 +1,21 @@
+import fse from "fs-extra";
+
+const source = process.argv[2];
+const dest = process.argv[3];
+
+const ignore = [
+  ".env",
+  "node_modules",
+  ".idea",
+  ".log",
+  ".vscode",
+  ".w3",
+  ".rs.bk",
+  ".lock",
+];
+
+fse.copySync(source, dest, {
+  filter: function (path) {
+    return !ignore.some((ignorePath) => path.endsWith(ignorePath));
+  },
+});

packages/cli/lang/en.json

@@ -13,6 +13,44 @@
   "commands_build_options_t": "Use the development server's ENS instance",
   "commands_build_options_w": "Automatically rebuild when changes are made (default: false)",
   "commands_build_options_v": "Verbose output (default: false)",
+  "commands_infra_description": "Manage infrastructure for your Web3API",
+  "commands_infra_actions_subtitle": "Infra allows you to execute the following commands:",
+  "commands_infra_options_options": "options",
+  "commands_infra_options_manifest": "manifest",
+  "commands_infra_options_i_node": "node",
+  "commands_infra_options_h": "Show usage information",
+  "commands_infra_options_m": "Use only specified modules",
+  "commands_infra_options_v": "Verbose output (default: false)",
+  "commands_infra_options_d": "Run in detached mode",
+  "commands_infra_moduleName": "module-name",
+  "commands_infra_manifestPath": "manifest-path",
+  "commands_infra_manifestPathDescription": "Infra Manifest path",
+  "commands_infra_command_up": "Start Web3API infrastructure",
+  "commands_infra_command_down": "Stop Web3API infrastructure",
+  "commands_infra_command_vars": "Show Web3API infrastructure's required .env variables",
+  "commands_infra_command_config": "Validate and display Web3API infrastructure's bundled docker-compose manifest",
+  "commands_infra_error_never": "This should never happen...",
+  "commands_infra_error_noCommand": "No command given",
+  "commands_infra_error_unrecognizedCommand": "Unrecognized command: {command}",
+  "commands_infra_error_noModulesMatch": "{modules} did not match any module names",
+  "commands_infra_error_noModulesDeclared": "No modules declared",
+  "commands_infra_modulesUsed_text": "Using infra modules",
+  "commands_infra_options_command": "command",
+  "commands_infra_options_start": "Startup the test env",
+  "commands_infra_options_stop": "Shutdown the test env",
+  "commands_infra_options_env_vars": "Show environment variables",
+  "commands_infra_shutdown_error": "Failed to shutdown test environment",
+  "commands_infra_shutdown_text": "Shutting down test environment...",
+  "commands_infra_shutdown_warning": "Warning shutting down test environment",
+  "commands_infra_startup_error": "Failed to start test environment",
+  "commands_infra_startup_text": "Starting test environment...",
+  "commands_infra_startup_warning": "Warning starting test environment",
+  "commands_infra_vars_text": "Extracting environment variables",
+  "commands_infra_vars_warning": "Warning extracting environment variables",
+  "commands_infra_vars_error": "Error extracting environment variables",
+  "commands_infra_manifest_text": "Aggregating manifest",
+  "commands_infra_manifest_warning": "Warning aggregating manifest",
+  "commands_infra_manifest_error": "Error aggregating manifest",
   "commands_build_uriViewers": "URI Viewers",
   "commands_deploy_description": "Deploys/Publishes a Web3API",
   "commands_deploy_options_options": "options",
@@ -140,7 +178,7 @@
   "commands_query_error_noApi": "API needs to be initialized",
   "commands_query_error_readFail": "Failed to read query {query}",
   "commands_query_error_noRecipeScriptFound": "Recipe script not found at path: {path}",
-  "commands_query_error_noTestEnvFound": "w3 test-env not found, please run 'w3 test-env up'",
+  "commands_query_error_noTestEnvFound": "w3 test-env not found, please run 'w3 infra up --modules=eth-ens-ipfs'",
   "commands_testEnv_description": "Manage a test environment for Web3API",
   "commands_testEnv_error_never": "This should never happen...",
   "commands_testEnv_error_noCommand": "No command given",
@@ -226,5 +264,6 @@
   "lib_watcher_alreadyWatching": "Watcher session is already in progress. Directory: {dir}",
   "lib_wasm_rust_invalidModule": "Module paths must point to Cargo.toml files. Found: {path}",
   "lib_docker_invalidImageId": "Invalid docker image ID returned: {imageId}",
-  "lib_docker_noInstall": "Docker executable not found in PATH"
+  "lib_docker_noInstall": "Docker executable not found in PATH",
+  "lib_infra_unrecognizedModule": "Unrecognized modules: {modules}. Default modules: {defaultModules}"
 }

packages/cli/lang/es.json

@@ -15,6 +15,44 @@
   "commands_build_options_w": "Automatically rebuild when changes are made (default: false)",
   "commands_build_options_v": "Verbose output (default: false)",
   "commands_build_uriViewers": "URI Viewers",
+  "commands_infra_description": "Manage infrastructure for your Web3API",
+  "commands_infra_actions_subtitle": "Infra allows you to execute the following commands:",
+  "commands_infra_options_options": "options",
+  "commands_infra_options_manifest": "manifest",
+  "commands_infra_options_i_node": "node",
+  "commands_infra_options_h": "Show usage information",
+  "commands_infra_options_m": "Use only specified modules",
+  "commands_infra_options_v": "Verbose output (default: false)",
+  "commands_infra_options_d": "Run in detached mode",
+  "commands_infra_moduleName": "module-name",
+  "commands_infra_manifestPath": "manifest-path",
+  "commands_infra_manifestPathDescription": "Infra Manifest path",
+  "commands_infra_command_up": "Start Web3API infrastructure",
+  "commands_infra_command_down": "Stop Web3API infrastructure",
+  "commands_infra_command_vars": "Show Web3API infrastructure's required .env variables",
+  "commands_infra_command_config": "Validate and display Web3API infrastructure's bundled docker-compose manifest",
+  "commands_infra_error_never": "This should never happen...",
+  "commands_infra_error_noCommand": "No command given",
+  "commands_infra_error_unrecognizedCommand": "Unrecognized command: {command}",
+  "commands_infra_error_noModulesMatch": "{modules} did not match any module names",
+  "commands_infra_error_noModulesDeclared": "No modules declared",
+  "commands_infra_modulesUsed_text": "Using infra modules",
+  "commands_infra_options_command": "command",
+  "commands_infra_options_start": "Startup the test env",
+  "commands_infra_options_stop": "Shutdown the test env",
+  "commands_infra_options_env_vars": "Show environment variables",
+  "commands_infra_shutdown_error": "Failed to shutdown test environment",
+  "commands_infra_shutdown_text": "Shutting down test environment...",
+  "commands_infra_shutdown_warning": "Warning shutting down test environment",
+  "commands_infra_startup_error": "Failed to start test environment",
+  "commands_infra_startup_text": "Starting test environment...",
+  "commands_infra_startup_warning": "Warning starting test environment",
+  "commands_infra_vars_text": "Extracting environment variables",
+  "commands_infra_vars_warning": "Warning extracting environment variables",
+  "commands_infra_vars_error": "Error extracting environment variables",
+  "commands_infra_manifest_text": "Aggregating manifest",
+  "commands_infra_manifest_warning": "Warning aggregating manifest",
+  "commands_infra_manifest_error": "Error aggregating manifest",
   "commands_deploy_description": "Deploys/Publishes a Web3API",
   "commands_deploy_options_options": "options",
   "commands_deploy_options_o_path": "path",
@@ -141,7 +179,7 @@
   "commands_query_error_noApi": "API needs to be initialized",
   "commands_query_error_readFail": "Failed to read query {query}",
   "commands_query_error_noRecipeScriptFound": "Recipe script not found at path: {path}",
-  "commands_query_error_noTestEnvFound": "w3 test-env not found, please run 'w3 test-env up'",
+  "commands_query_error_noTestEnvFound": "w3 test-env not found, please run 'w3 infra up --modules=eth-ens-ipfs'",
   "commands_testEnv_description": "Manage a test environment for Web3API",
   "commands_testEnv_error_never": "This should never happen...",
   "commands_testEnv_error_noCommand": "No command given",
@@ -227,5 +265,6 @@
   "lib_watcher_alreadyWatching": "Watcher session is already in progress. Directory: {dir}",
   "lib_wasm_rust_invalidModule": "Module paths must point to Cargo.toml files. Found: {path}",
   "lib_docker_invalidImageId": "Invalid docker image ID returned: {imageId}",
-  "lib_docker_noInstall": "Docker executable not found in PATH"
+  "lib_docker_noInstall": "Docker executable not found in PATH",
+  "lib_infra_unrecognizedModule": "Unrecognized modules: {modules}. Default modules: {defaultModules}"
 }

packages/cli/package.json

@@ -16,11 +16,12 @@
     "w3": "bin/w3"
   },
   "scripts": {
-    "build": "rimraf ./build && tsc --project tsconfig.build.json && yarn build:envs && yarn build:deployers",
-    "build:envs": "copyfiles ./src/lib/build-envs/**/**/* ./build/lib/build-envs/ -u 3",
-    "build:deployers": "copyfiles ./src/lib/deployers/**/web3api.deploy.ext.json ./build/lib/deployers -u 3",
+    "build": "rimraf ./build && tsc --project tsconfig.build.json && yarn build:build-images && yarn build:deploy-modules && yarn build:infra-modules",
+    "build:build-images": "copyfiles ./src/lib/defaults/build-images/**/**/* ./build/lib/defaults/build-images/ -u 4",
+    "build:deploy-modules": "copyfiles ./src/lib/defaults/deploy-modules/**/web3api.deploy.ext.json ./build/lib/defaults/deploy-modules -u 4",
+    "build:infra-modules": "ts-node ./copyfiles ./src/lib/defaults/infra-modules ./build/lib/defaults/infra-modules",
     "prebuild": "ts-node ./scripts/generateIntlTypes.ts",
-    "build:fast": "rimraf ./build && tsc --project tsconfig.build.json && yarn build:envs",
+    "build:fast": "rimraf ./build && tsc --project tsconfig.build.json",
     "lint": "eslint --color -c ../../.eslintrc.js .",
     "test": "cross-env TEST=true jest --passWithNoTests --runInBand --verbose",
     "test:ci": "cross-env TEST=true jest --passWithNoTests --runInBand --verbose",
@@ -30,11 +31,11 @@
     "colors": "1.4.0"
   },
   "dependencies": {
-    "@ensdomains/ensjs": "2.0.1",
     "@formatjs/intl": "1.8.2",
+    "@ethersproject/wallet": "5.6.2",
+    "@ethersproject/providers": "5.6.8",
     "@web3api/asyncify-js": "0.0.1-prealpha.83",
     "@web3api/client-js": "0.0.1-prealpha.83",
-    "@web3api/client-test-env": "0.0.1-prealpha.83",
     "@web3api/core-js": "0.0.1-prealpha.83",
     "@web3api/ens-plugin-js": "0.0.1-prealpha.83",
     "@web3api/ethereum-plugin-js": "0.0.1-prealpha.83",
@@ -43,18 +44,19 @@
     "@web3api/schema-bind": "0.0.1-prealpha.83",
     "@web3api/schema-compose": "0.0.1-prealpha.83",
     "@web3api/schema-parse": "0.0.1-prealpha.83",
+    "@web3api/test-env-js": "0.0.1-prealpha.83",
     "assemblyscript": "0.19.1",
     "axios": "0.21.2",
     "chalk": "4.1.0",
     "chokidar": "3.5.1",
     "commander": "9.2.0",
     "content-hash": "2.5.2",
     "copyfiles": "2.4.1",
-    "ethers": "5.0.7",
     "fs-extra": "9.0.1",
     "gluegun": "4.6.1",
     "graphql-tag": "2.11.0",
     "ipfs-http-client": "48.1.3",
+    "docker-compose": "0.23.17",
     "js-yaml": "3.14.0",
     "jsonschema": "1.4.0",
     "mustache": "4.0.1",
@@ -72,7 +74,6 @@
     "@types/node": "12.12.26",
     "@types/prettier": "2.6.0",
     "@types/rimraf": "3.0.0",
-    "@web3api/test-env-js": "0.0.1-prealpha.83",
     "cross-env": "7.0.3",
     "dir-compare": "3.3.0",
     "eslint-plugin-formatjs": "2.12.7",

packages/cli/src/__tests__/e2e/build.spec.ts

@@ -60,16 +60,16 @@ describe("e2e tests for build command", () => {
     expect(output).toBe("")
   });
 
-  test("Adds uuid-v4 suffix to build-env image if no build manifest specified", async () => {
+  test("Adds uuid-v4 suffix to build image if no build manifest specified", async () => {
     const projectRoot = getTestCaseDir(0);
     const project = new Web3ApiProject({
-      rootCacheDir: projectRoot,
+      rootDir: projectRoot,
       web3apiManifestPath: path.join(projectRoot, "web3api.yaml")
     });
 
-    await project.cacheDefaultBuildManifestFiles();
+    await project.cacheDefaultBuildImage();
 
-    const cacheBuildEnvPath = path.join(projectRoot, ".w3/web3api/build/env")
+    const cacheBuildEnvPath = path.join(projectRoot, ".w3/web3api/build/image")
     const cachedBuildManifest = await loadBuildManifest(
       path.join(cacheBuildEnvPath, "web3api.build.yaml")
     );

packages/cli/src/__tests__/e2e/deploy.spec.ts

@@ -4,12 +4,13 @@ import {
   initTestEnvironment,
   runCLI,
   stopTestEnvironment,
+  ensAddresses,
+  providers
 } from "@web3api/test-env-js";
 import { GetPathToCliTestFiles } from "@web3api/test-cases";
-import axios from "axios";
 import { Web3ApiClient } from "@web3api/client-js";
 import { ethereumPlugin } from "@web3api/ethereum-plugin-js";
-import { Wallet } from "ethers";
+import { Wallet } from "@ethersproject/wallet";
 import path from "path";
 import fs from "fs";
 
@@ -33,12 +34,12 @@ const testCaseRoot = path.join(GetPathToCliTestFiles(), "api/deploy");
     path.join(testCaseRoot, testCases[index]);
 
 const setup = async (domainNames: string[]) => {
-  const { ethereum } = await initTestEnvironment();
-  const { data } = await axios.get("http://localhost:4040/deploy-ens");
+  await stopTestEnvironment();
+  await initTestEnvironment();
 
-  const ensAddress = data.ensAddress
-  const resolverAddress = data.resolverAddress
-  const registrarAddress = data.registrarAddress
+  const ensAddress = ensAddresses.ensAddress
+  const resolverAddress = ensAddresses.resolverAddress
+  const registrarAddress = ensAddresses.registrarAddress
   const signer = new Wallet("0x4f3edf983ac636a65a842ce7c78d9aa706d3b113bce9c46f30d7d21715b23b1d");
 
   // Setup environment variables
@@ -49,15 +50,14 @@ const setup = async (domainNames: string[]) => {
   };
 
   const ethereumPluginUri = "w3://ens/ethereum.web3api.eth"
-
   const client = new Web3ApiClient({
     plugins: [
       {
         uri: ethereumPluginUri,
         plugin: ethereumPlugin({
           networks: {
             testnet: {
-              provider: ethereum,
+              provider: providers.ethereum,
               signer
             }
           },
@@ -73,7 +73,7 @@ const setup = async (domainNames: string[]) => {
   )}`;
 
   for await (const domainName of domainNames) {
-    await client.invoke<{ hash: string }>({
+    const result = await client.invoke({
       uri: ensWrapperUri,
       module: "mutation",
       method: "registerDomainAndSubdomainsRecursively",
@@ -88,7 +88,13 @@ const setup = async (domainNames: string[]) => {
           networkNameOrChainId: "testnet",
         },
       },
-    })
+    });
+
+    if (result.error) {
+      throw Error(
+        `Failed to register ${domainName}: ${result.error.message}`
+      );
+    }
   }
 }
 
@@ -105,11 +111,13 @@ describe("e2e tests for deploy command", () => {
         },
       );
     }
+
+    await new Promise((resolve) => setTimeout(resolve, 5000));
   });
 
   afterAll(async () => {
     await stopTestEnvironment();
-  })
+  });
 
   test("Should show help text", async () => {
     const { exitCode: code, stdout: output, stderr: error } = await runCLI(
@@ -126,7 +134,7 @@ describe("e2e tests for deploy command", () => {
   });
 
   test("Successfully deploys the project", async () => {
-    const { exitCode: code, stdout: output } = await runCLI(
+    const { exitCode: code, stdout: output, stderr: error } = await runCLI(
       {
         args: ["deploy"],
         cwd: getTestCaseDir(0),
@@ -137,19 +145,14 @@ describe("e2e tests for deploy command", () => {
 
     const sanitizedOutput = clearStyle(output);
 
+    expect(error).toBeFalsy();
     expect(code).toEqual(0);
     expect(sanitizedOutput).toContain(
       "Successfully executed stage 'ipfs_deploy'"
     );
     expect(sanitizedOutput).toContain(
       "Successfully executed stage 'from_deploy'"
     );
-    expect(sanitizedOutput).toContain(
-      "Successfully executed stage 'from_deploy2'"
-    );
-    expect(sanitizedOutput).toContain(
-      "Successfully executed stage 'from_uri'"
-    );
   });
 
   test("Should show warning if no manifest ext is found in deploy package", async () => {

packages/cli/src/__tests__/e2e/help.spec.ts

@@ -2,6 +2,7 @@ import { clearStyle, w3Cli } from "./utils";
 
 import { runCLI } from "@web3api/test-env-js";
 
+
 const HELP = `Usage: w3 [options] [command]
 
 Options:
@@ -15,7 +16,7 @@ Commands:
   deploy|d [options]          Deploys/Publishes a Web3API
   plugin|p                    Build/generate types for the plugin
   query|q [options] <recipe>  Query Web3APIs using recipe scripts
-  test-env|t                  Manage a test environment for Web3API
+  infra|i [options] <action>  Manage infrastructure for your Web3API
   help [command]              display help for command
 `;