build: use tsc project references, npm v7 workspaces hyperledger-cact…

…i#312 hyperledger-cacti#973 1. Migrates the build from lerna to the built-in Typescript compiler project references feature meaning that instead of running the typescript compiler in a separate, dedicated process for each package it only runs a single process of tsc (typescript compiler) which processes all the packages. This provides a speed boost and also finally allows IDEs (integrated development environments) to better index the source code and the build artifacts together in order to provide up to date compiler errors (previously if you had done a full rebuild of the souce code you also needed to reload your VSCode window for example in order to jolt it into purging it's code cache and stop giving false negative errors/warnings about missing packages) 2. Moves the project build to npm v7 workspaces which means that we will no longer need lerna to bootstrap dependencies by sym- linking them together for the sybling packages. Instead npm will take care of all this which provices a significant speedup in the time the configure script takes to run. 3. Also migrates all front-end (Ionic+Angular) packages to the latest version of Angular at the time of this writing (v12) because the new build system changes mandated this (Angular dependencies could not be resolved properly because of us using npm workspaces) 4. Added an FAQ entry to show developers how to increase the Linux kernel's open file limits (per process) which was necessary because on WSL 2 Ubuntu 20.04 the npm install process was failing when the limit was on 4096 files by default. 5. The webpack config had to be migrated to the new webpack/cli package (major version upgrade had breaking changes as expected) Again, this was necessary because of the cascading effects all stemming from us migrating to the npm workspaces. Why conflate all these different issues into a single commit/PR? Because as it turned out you could not make a single change without also applying all the others (which was not know at the time of the issues for these tasks being created). Fixes hyperledger-cacti#312 Fixes hyperledger-cacti#973 Signed-off-by: Peter Somogyvari <peter.somogyvari@accenture.com>
Leeyoungone · Jul 21, 2021 · b3b14df · b3b14df
1 parent 260990b
commit b3b14df
Show file tree

Hide file tree

Showing 105 changed files with 164,639 additions and 91,808 deletions.
diff --git a/.cspell.json b/.cspell.json
@@ -4,85 +4,100 @@
     "minWordLength": 4,
     "allowCompoundWords": true,
     "words": [
+        "adminpw",
+        "Albertirsa",
+        "ALLFORTX",
+        "ANYFORTX",
+        "APIV",
+        "approveformyorg",
+        "Authz",
+        "authzn",
         "Besu",
+        "Bools",
+        "cafile",
+        "caio",
+        "cccs",
+        "ccid",
+        "cids",
         "Corda",
-        "HyperLedger",
-        "DockerOde",
-        "Keychain",
-        "Keycloak",
         "Cordapp",
-        "Healthcheck",
-        "uuidv",
-        "Xdai",
-        "OpenAPI",
+        "dclm",
+        "DHTAPI",
+        "DockerOde",
+        "ealen",
         "Errorf",
-        "Oidc",
-        "Authz",
-        "Bools",
-        "Secp",
-        "openethereum",
-        "NODETXPOOLACK",
-        "LEDGERBLOCKACK",
+        "escc",
+        "execa",
+        "faio",
+        "fsouza",
         "GETHKEYCHAINPASSWORD",
-        "NETWORKSCOPEALLFORTX",
-        "tlsca",
+        "ghcr",
+        "gopath",
         "grpc",
         "grpcs",
-        "MSPCONFIGPATH",
-        "LOCALMSPID",
-        "organisation",
-        "notok",
-        "txqueue",
-        "tlscacerts",
-        "dclm",
-        "SPDX",
-        "mychannel",
-        "gopath",
-        "myvolume",
-        "authzn",
-        "supervisord",
+        "hashicorp",
+        "Healthcheck",
+        "HTLC",
+        "Htlc",
+        "htlc",
+        "HyperLedger",
         "ipaddress",
+        "ipfs",
+        "jboss",
+        "JORDI",
+        "Keychain",
+        "Keycloak",
+        "Knetic",
+        "LEDGERBLOCKACK",
+        "LOCALMSPID",
+        "miekg",
+        "mitchellh",
+        "MSPCONFIGPATH",
+        "MSPID",
         "MSPIDSCOPEALLFORTX",
-        "ANYFORTX",
-        "MSPIDSCOPEALLFORTX",
-        "NETWORKSCOPEANYFORTX",
         "MSPIDSCOPEANYFORTX",
-        "MSPID",
-        "ALLFORTX",
-        "cccs",
-        "svcs",
-        "vscc",
-        "escc",
-        "faio",
-        "hashicorp",
-        "cafile",
-        "supervisorctl",
-        "onsi",
         "Mtls",
-        "APIV",
+        "myapp",
+        "mychannel",
+        "myroot",
+        "myvolume",
         "Nerc",
-        "ccid",
-        "adminpw",
-        "caio",
+        "NETWORKSCOPEALLFORTX",
+        "NETWORKSCOPEANYFORTX",
+        "NODETXPOOLACK",
+        "notok",
+        "Oidc",
+        "onsi",
+        "OpenAPI",
+        "openethereum",
+        "organisation",
         "protos",
-        "Unmarshal",
-        "myroot",
-        "ealen",
-        "approveformyorg",
-        "jboss",
+        "RUSTC",
+        "Secp",
+        "socketio",
+        "SPDX",
         "Sprintf",
+        "stretchr",
+        "supervisorctl",
+        "supervisord",
+        "svcs",
+        "sykesm",
         "thream",
-        "socketio",
-        "Albertirsa",
-        "HTLC",
-        "Htlc",
-        "htlc"
+        "tlsca",
+        "tlscacerts",
+        "txqueue",
+        "Unmarshal",
+        "uuidv",
+        "vscc",
+        "wasm",
+        "Xdai"
     ],
     "dictionaries": [
-        "typescript,node,npm,go"
+        "typescript,node,npm,go,rust"
     ],
     "ignorePaths": [
         "node_modules/**",
-        "**/build/**"
+        "**/build/**",
+        "**/src/main/typescript/generated/**"
     ]
 }
diff --git a/.dcilintignore b/.dcilintignore
@@ -1,4 +1,2 @@
-# TSLint configuration parameter
-packages/cactus-cockpit/tslint.json
 # Besu CLI argument
 tools/docker/besu-all-in-one/supervisord.conf
diff --git a/.eslintrc.js b/.eslintrc.js
@@ -6,7 +6,7 @@ module.exports = {
   },
   extends: [
     "plugin:@typescript-eslint/recommended", // Uses the recommended rules from the @typescript-eslint/eslint-plugin
-    "prettier/@typescript-eslint", // Uses eslint-config-prettier to disable ESLint rules from @typescript-eslint/eslint-plugin that would conflict with prettier
+    "prettier", // Uses eslint-config-prettier to disable ESLint rules from @typescript-eslint/eslint-plugin that would conflict with prettier
     "plugin:prettier/recommended", // Enables eslint-plugin-prettier and eslint-config-prettier. This will display prettier errors as ESLint errors. Make sure this is always the last configuration in the extends array.
   ],
   rules: {

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -19,23 +19,23 @@ jobs:
       max-parallel: 1024
       matrix:
         os: [ubuntu-20.04]
-        node-version: [v12.13.0, v14.15.1, v16.2.0]
+        node-version: [v12.22.3, v14.15.1, v16.2.0]
         experimental: [false]
         # include:
         #
         # # FIXME macOS does not work due to lack of docker support in GHA.
         # https://github.saobby.my.eu.orgmunity/t/why-is-docker-not-installed-on-macos/17017
         #   - os: macos-11.0 # macOS Big Sur 11.0
-        #     node-version: v12.13.0
+        #     node-version: v12.22.3
         #     experimental: true
         #   - os: macos-10.15 # macOS Catalina 10.15
-        #     node-version: v12.13.0
+        #     node-version: v12.22.3
         #     experimental: true
         #
         # # FIXME
         # https://github.com/hyperledger/cactus/issues/171
         #   - os: windows-2019 # Windows Server 2019
-        #     node-version: v12.13.0
+        #     node-version: v12.22.3
         #     experimental: true
 
     steps:

diff --git a/.gitignore b/.gitignore
@@ -44,4 +44,6 @@ cactus-openapi-spec-*.json
 *.log
 build/
 .gradle/
-site/
+site/
+
+.build-cache/*.tsbuildinfo
diff --git a/.lintstagedrc.json b/.lintstagedrc.json
@@ -0,0 +1,6 @@
+{
+  "*/*/src/**/*.{js,ts}": [
+    "eslint --fix",
+    "cspell --no-must-find-files --config .cspell.json"
+  ]
+}
diff --git a/.npmignore b/.npmignore
@@ -0,0 +1 @@
+# empty npmignore to ensure all required files (e.g., in the dist folder) are published by npm
diff --git a/BUILD.md b/BUILD.md
@@ -35,7 +35,7 @@ This is a video guide to setup Hyperledger Cactus on your local machine.
 [![IMAGE ALT TEXT HERE](https://img.youtube.com/vi/n_HiEwgzPsM/0.jpg)](https://www.youtube.com/watch?v=n_HiEwgzPsM)
 
 ### Compiling all packages
- 
+
 [![IMAGE ALT TEXT HERE](https://img.youtube.com/vi/5v82MAHPQmM/0.jpg)](https://www.youtube.com/watch?v=5v82MAHPQmM)
 
 ### Testing all packages
@@ -87,9 +87,9 @@ The `npm run watch` script in action:
 
 * Install OS level dependencies:
   * Windows Only
-    * WSL1 or WSL2 or any virtual machine running Ubuntu LTS
+    * WSL2 or any virtual machine running Ubuntu 20.04 LTS
   * Git
-  * NodeJS 12 or newer LTS (we recommend using nvm if available for your OS)
+  * NodeJS 16, npm 7 (we recommend using the Node Version Manager (nvm) if available for your OS)
   * Docker Engine
   * Docker Compose
 
@@ -128,7 +128,7 @@ For example you can *run a ledger contract deployment test* via the
 REST API with this command:
 
 ```sh
-npx tap --ts --timeout=600 packages/cactus-test-plugin-ledger-connector-quorum/src/test/typescript/integration/plugin-ledger-connector-quorum/deploy-contract/deploy-contract-via-web-service.ts
+npx tap --ts --timeout=600 packages/cactus-test-plugin-ledger-connector-quorum/src/test/typescript/integration/plugin-ledger-connector-quorum/deploy-contract/deploy-contract-via-web-service.test.ts
 ```
 
 *You can also start the API server* and verify more complex scenarios with an
@@ -147,11 +147,10 @@ The most interesting part of the `.config.json` file is the plugins array which
 takes a list of plugin package names and their options (which can be anything
 that you can fit into a generic JSON object).
 
-Notice that to include a plugin, all you need is specify it's npm package name
-(and ensure that said package is actually installed). This is important since
-it allows you to have your own plugins in their respective, independent Github
-repositories and npm packages where you do not have to seek explicit approval
-from the Cactus maintainers to create/maintain your plugin at all.
+Notice that to include a plugin, all you need is specify it's npm package name.
+This is important since it allows you to have your own plugins in their respective,
+independent Github repositories and npm packages where you do not have to seek
+explicit approval from the Cactus maintainers to create/maintain your plugin at all.
 
 Once you are satisfied with the `.config.json` file's contents you can just:
 
@@ -173,14 +172,14 @@ from that state.
 
 At this point, with the running API server, you can
 * Test the REST API directly with tools like cURL or Postman
-* Develop your own applications against it with the `Cactus SDK`
+* Develop your own applications against it with the `Cactus API Client(s)`
 * Create and test your own plugins
 
 
 #### Random Windows specific issues not covered here
 
-We recommend you use WSL or WSL2 or any Linux VM. We test most frequently on
-Ubuntu LTS which at the time of this writing means 18.04 and/or 20.04.
+We recommend that you use WSL2 or any Linux VM (or bare metal).
+We test most frequently on Ubuntu 20.04 LTS
 
 ## Build Script Decision Tree
 

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -17,8 +17,7 @@
   - [All-In-One Docker Images for Ledger Connector Plugins](#all-in-one-docker-images-for-ledger-connector-plugins)
     - [Test Automation of Ledger Plugins](#test-automation-of-ledger-plugins)
   - [Building the SDK](#building-the-sdk)
-  - [Adding a new public npm dependency to one of the packages:](#adding-a-new-public-npm-dependency-to-one-of-the-packages)
-  - [Adding a sibling package npm dependency to one of the packages:](#adding-a-sibling-package-npm-dependency-to-one-of-the-packages)
+  - [Adding new dependencies:](#adding-new-dependencies)
   - [Reload VSCode Window After Adding Dependencies](#reload-vscode-window-after-adding-dependencies)
   - [On Reproducible Builds](#on-reproducible-builds)
 
@@ -493,7 +492,7 @@ npx tap --ts --jobs=1 --timeout=600 \"./\"
 ### Building the SDK
 
 You do not need to do anything special to have the SDK sources generated and
-compiled. It is all part of the `npm run build` task which you can run yourself
+compiled. It is all part of the `npm run build:dev:backend` task which you can run yourself
 or as part of the CI script (`./tools/ci.sh`).
 
 The SDK is itself just another package named `sdk` and can be dependend on by
@@ -504,33 +503,24 @@ browser and also NodeJS environments. This is very important as we do not wish
 to maintain two (or more) separate SDK codebases and we also want as much of it
 being generated automatically as possible (currently this is close to 100%).
 
-### Adding a new public npm dependency to one of the packages:
+### Adding new dependencies:
 
-For example web3 can be added as a dependency to the besu ledger connector plugin's package this way:
+Example:
 
 ```sh
-npx lerna add web3@latest --scope '*/*plugin-ledger-connector-besu' --exact # [--dev] [--peer]
+# Adds "got" as a dependency to the cactus common package
+# (which resides under the path of ./packages/cactus-common)
+npm install got --save-exact --workspace ./packages/cactus-common
 ```
 
-If you are adding a development dependency you can use the `--dev` option and `--peer` for a peer dependency.
+You need to know which package of the monorepo will be using the package and then
+run the `npm install` command with an additional parameter specifying the directory
+of that package. See [adding-dependencies-to-a-workspace](https://docs.npmjs.com/cli/v7/using-npm/workspaces#adding-dependencies-to-a-workspace) from the official npm documentation
+for further details and examples.
 
-After running any `lerna add` command you might need to [Reload VSCode Window After Adding Dependencies](#reload-vscode-window-after-adding-dependencies)
+After adding new dependencies, you might need to [Reload VSCode Window After Adding Dependencies](#reload-vscode-window-after-adding-dependencies)
 
-### Adding a sibling package npm dependency to one of the packages:
-
-For example the `cactus-test-tooling` can be added as a dev dependency to the besu ledger connector plugin's package this way:
-
-```sh
-npx lerna add @hyperledger/cactus-test-tooling --scope '*/*plugin-ledger-connector-besu' --exact --dev
-```
-
-Or add the common library to allow you the usage of the logger for example:
-
-```sh
-npx lerna add @hyperledger/cactus-common --scope '*/*plugin-ledger-connector-quorum' --exact --dev
-```
-
-After running any `lerna add` command you might need to [Reload VSCode Window After Adding Dependencies](#reload-vscode-window-after-adding-dependencies)
+> **Always specify the `--save-exact` when installing new dependencies to ensure [reproducible builds](https://reproducible-builds.org/)**
 
 ### Reload VSCode Window After Adding Dependencies
 

diff --git a/FAQ.md b/FAQ.md
@@ -1,5 +1,6 @@
 # Frequently Asked Questions - Hyperledger Cactus <!-- omit in toc -->
 
+- [npm install fails with `code EMFILE` (syscall connect)](#npm-install-fails-with-code-emfile-syscall-connect)
 - [Where does the project name come from?](#where-does-the-project-name-come-from)
 - [Re-building the code after I change it slow. What to do?](#re-building-the-code-after-i-change-it-slow-what-to-do)
 - [The watch script fails with `Internal watch failed: watch ENOSPC`](#the-watch-script-fails-with-internal-watch-failed-watch-enospc)
@@ -10,6 +11,20 @@
 - [Why do all the tests bind the HTTP/S listeners to a random port?](#why-do-all-the-tests-bind-the-https-listeners-to-a-random-port)
 - [HTTP requests sent with Postman/curl/etc. hang if I'm debugging a test case with VSCode, why?](#http-requests-sent-with-postmancurletc-hang-if-im-debugging-a-test-case-with-vscode-why)
 
+## npm install fails with `code EMFILE` (syscall connect)
+
+If you encounter this error it happens because the number of open files permitted
+by your operating system is lower than the number of files npm needs to have open
+in order to finish installing your dependencies.
+
+You can most likely solve this problem (assuming you are running on the recommended
+operating system distribution) by running the command below on the terminal which
+will increase said limit to 200k from the default () 4096.
+
+```sh
+echo fs.file-max=200000 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p
+```
+
 ## Where does the project name come from?
 
 It was the first one on our proposed list of names that satisfied the following criteria:

diff --git a/examples/cactus-example-carbon-accounting-backend/package-lock.json b/examples/cactus-example-carbon-accounting-backend/package-lock.json
-Original file line number
+Diff line change
@@ Expand Up / @@ -44,4 +44,6 @@ cactus-openapi-spec-*.json @@
     *.log
     build/
     .gradle/
-    site/
+    site/
+    .build-cache/*.tsbuildinfo
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		# empty npmignore to ensure all required files (e.g., in the dist folder) are published by npm