feat(persistence-fabric): rewrite the plugin

- Rewrite fabric persistence plugin to match persistence ethereum plugin methods and behavior. - Change DB schema for persistence. - Add tests and sample data to the new plugin. Add them to project CI. Depends on hyperledger-cacti#3298 Signed-off-by: Michal Bajer <michal.bajer@fujitsu.com>
outSH · Jul 18, 2024 · aacae5e · aacae5e
1 parent 497ea32
commit aacae5e
Show file tree

Hide file tree

Showing 34 changed files with 4,645 additions and 1,669 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
diff --git a/packages/cactus-plugin-persistence-fabric/CHANGELOG.md b/packages/cactus-plugin-persistence-fabric/CHANGELOG.md
diff --git a/packages/cactus-plugin-persistence-fabric/Dockerfile b/packages/cactus-plugin-persistence-fabric/Dockerfile
@@ -0,0 +1,14 @@
+FROM node:16
+
+ENV PACKAGE_PATH=/opt/cactus-plugin-persistence-fabric
+
+WORKDIR ${PACKAGE_PATH}
+
+# CMake is required by one of npm dependencies (install other packages in this step as well in the future)
+RUN apt-get update && apt-get install -y cmake && rm -rf /var/lib/apt/lists/*
+
+COPY ./dist/yarn.lock ./package.json ./
+RUN yarn install --production --ignore-engines --non-interactive --cache-folder ./.yarnCache && \
+ rm -rf ./.yarnCache
+
+COPY ./dist ./dist
diff --git a/packages/cactus-plugin-persistence-fabric/README.md b/packages/cactus-plugin-persistence-fabric/README.md
@@ -1,9 +1,7 @@
 # `@hyperledger/cactus-plugin-persistence-fabric`
 
-This plugin allows `Cacti` to persist Fabric Block general data and basic information about transactions into some storage (currently to a `PostgreSQL` database, but this concept can be extended further).
+This plugin allows `Cacti` to persist Hyperledger Fabric data into some storage (currently to a `PostgreSQL` database, but this concept can be extended further).
 Data in the database can later be analyzed and viewed in a GUI tool.
-GUI tool is in project root directory of Cacti project in GUI folder. cacti/packages/cacti-ledger-browser
-
 
 ## Summary
 
@@ -17,9 +15,9 @@ GUI tool is in project root directory of Cacti project in GUI folder. cacti/pack
 
 ## Remarks
 
-- This plugin was only tested with small test ledgers. Running it to synchronize with old ledgers will take a lot of time.
+- This plugin was only tested with small Fabric ledgers. Running it to archive and monitor large ledgers is not recommended yet.
 - For now, the database schema is not considered public and can change over time (i.e., writing own application that reads data directly from the database is discouraged).
-- All the methods must be called directly on the plugin instance for now.
+- Only `status` endpoint is available, all the methods must be called directly on the plugin instance for now.
 
 ## Getting Started
 
@@ -35,65 +33,88 @@ yarn run configure
 
 ### Usage
 
-Instantiate a new `PluginPersistenceFabrickBlock` instance:
+Instantiate a new `PluginPersistenceFabric` instance:
 
 ```typescript
-import { PluginPersistenceFabric } from "../../../main/typescript/plugin-fabric-persistence-block";
+import { PluginPersistenceFabric } from "@hyperledger/cactus-plugin-persistence-fabric";
 import { v4 as uuidv4 } from "uuid";
 
-PluginInstance = new PluginPersistenceFabric({
- gatewayOptions,
+const persistencePlugin = new PluginPersistenceFabric({
  apiClient,
- logLevel: testLogLevel,
- instanceId: uuidv4(),
- connectionString:
- "postgresql://postgres:your-super-secret-and-long-postgres-password@localhost:5432/postgres",
+ logLevel: "info",
+ instanceId,
+ connectionString: "postgresql://postgres:your-super-secret-and-long-postgres-password@localhost:5432/postgres",,
+ channelName: "mychannel",
+ gatewayOptions: {
+ identity: signingCredential.keychainRef,
+ wallet: {
+ keychain: signingCredential,
+ },
+ },
 });
 
 // Initialize the connection to the DB
-PluginInstance.onPluginInit();
+await persistencePlugin.onPluginInit();
 ```
 
-Alternatively, import `PluginPersistenceFabric` from the plugin package and use it to create a plugin.
+Alternatively, import `PluginFactoryLedgerPersistence` from the plugin package and use it to create a plugin.
 
 ```typescript
-import { PluginPersistenceFabric } from "@hyperledger/plugin-fabric-persistence-block";
+import { PluginFactoryLedgerPersistence } from "@hyperledger/cactus-plugin-persistence-fabric";
 import { PluginImportType } from "@hyperledger/cactus-core-api";
 import { v4 as uuidv4 } from "uuid";
 
-const factory = new PluginFactoryPersistenceFabricBlock({
+const factory = new PluginFactoryLedgerPersistence({
  pluginImportType: PluginImportType.Local,
 });
 
 const persistencePlugin = await factory.create({
- instanceId: uuidv4(),
- apiClient: new SocketIOApiClient(apiConfigOptions),
+ apiClient,
  logLevel: "info",
- connectionString:
- "postgresql://postgres:your-super-secret-and-long-postgres-password@localhost:5432/postgres",
+ instanceId,
+ connectionString: "postgresql://postgres:your-super-secret-and-long-postgres-password@localhost:5432/postgres",,
+ channelName: "mychannel",
+ gatewayOptions: {
+ identity: signingCredential.keychainRef,
+ wallet: {
+ keychain: signingCredential,
+ },
+ },
 });
 
 // Initialize the connection to the DB
-persistencePlugin.onPluginInit();
+await persistencePlugin.onPluginInit();
 ```
 
-onPluginInit it also creates database structure if this is first time run
+You can use the persistent plugin to synchronize ledger state with the database. Here is a sample script that starts monitoring for new blocks:
 
-// Start synchronization with ledger.
-// To synchronize ledger
-
-you individually
+```typescript
+// Start monitoring new blocks.
+// Entire ledger is synchronized first with the DB (`syncAll` is called) so this operation can take a while on large ledgers!
+persistencePlugin.startMonitor((err) => {
+ reject(err);
+});
 
 // Show current status of the plugin
-PluginInstance.getStatus();
-
+persistencePlugin.getStatus();
 ```
 
+> See [plugin integration tests](./src/test/typescript/integration) for complete usage examples.
+
+### Building/running the container image locally
 
+In the Cacti project root say:
+
+```sh
+DOCKER_BUILDKIT=1 docker build ./packages/cactus-plugin-persistence-fabric/ -f ./packages/cactus-plugin-persistence-fabric/Dockerfile -t cactus-plugin-persistence-fabric
 ```
 
 ## Endpoints
 
+### StatusV1 (`/api/v1/plugins/@hyperledger/cactus-plugin-persistence-fabric/status`)
+
+- Returns status of the plugin (latest block read, failed blocks, is monitor running, etc...)
+
 ### Plugin Methods
 
 - Most of the plugin functionalities are currently not available through OpenAPI interface, please use direct method calls instead.
@@ -110,42 +131,28 @@ PluginInstance.getStatus();
 
 - Get status report of this instance of persistence plugin.
 
-#### `constinousBlocksSynchronization`
-
-- Start the block synchronization process. New blocks from the ledger will be parsed and pushed to the database.
-
-#### `continueBlocksSynchronization`
-
-- Start the block synchronization process. New blocks from the ledger will be parsed and pushed to the database. Should be used periodically.
+#### `startMonitor`
 
-#### `changeSynchronization`
+- Start the block monitoring process. New blocks from the ledger will be parsed and pushed to the database.
 
-- Stop the block synchronization process.
+#### `stopMonitor`
 
-#### `whichBlocksAreMissingInDdSimple`
+- Stop the block monitoring process.
 
-- Walk through all the blocks that could not be synchronized with the DB for some reasons and list them
+#### `syncFailedBlocks`
 
-#### `synchronizeOnlyMissedBlocks`
+- Walk through all the blocks that could not be synchronized with the DB for some reasons and try pushing them again.
 
-- Walk through all the blocks that are listed as not be synchronized with the DB for some reasons and try push them into DB from ledger.
-- can try many times to use this
+#### `syncAll`
 
-### `setLastBlockConsidered`
-
-- set the last block in ledger which we consider valid by our party and synchronize only to this point in ledger
- If some blocks above this number are already in database they will not be removed.
-
-#### `initialBlocksSynchronization`
-
-- Synchronize entire first N number of blocks of ledger state with the database. It is a good start and easy to check if everything is correctly set.
+- Synchronize entire ledger state with the database.
 
 ## Running the tests
 
-To run all the tests for this persistence fabric plugin to ensure it's working correctly execute the following from the root of the `cactus` project:
+To run all the tests for this persistence plugin to ensure it's working correctly execute the following from the root of the `cactus` project:
 
 ```sh
-npx jest cactus-plugin-fabric-persistence-block
+npx jest cactus-plugin-persistence-fabric
 ```
 
 ## Contributing
@@ -156,23 +163,19 @@ Please review [CONTRIBUTING.md](../../CONTRIBUTING.md) to get started.
 
 ### Quick plugin project walkthrough
 
-#### ./src/main/json/contract_abi
-
-- Contains reference token ABIs used to call and identify token transfers.
-
 #### `./src/main/json/openapi.json`
 
 - Contains OpenAPI definition.
 
 #### `./src/main/sql/schema.sql`
 
-- Database schema for Ethereum data.
+- Database schema for Fabric data.
 
 #### `./src/main/typescript/web-services`
 
-- Folder that contains web service endpoint definitions if present.
+- Folder that contains web service endpoint definitions.
 
-#### `./plugin-fabric-persistence-block`
+#### `./plugin-persistence-fabric`
 
 - Main persistent plugin logic file
 

diff --git a/packages/cactus-plugin-persistence-fabric/package.json b/packages/cactus-plugin-persistence-fabric/package.json
@@ -1,7 +1,7 @@
 {
  "name": "@hyperledger/cactus-plugin-persistence-fabric",
  "version": "2.0.0-rc.2",
- "description": "A web service plugin that provides management capabilities on a Cacti Plugin Fabric Block Persistence as a scrapper of blocks.",
+ "description": "Persistence plugin for Fabric ledgers to store data into a database.",
  "keywords": [
  "Hyperledger",
  "Cacti",
@@ -25,69 +25,55 @@
  },
  "contributors": [
  {
- "name": "Barnaba Pawelczak",
- "email": "barnaba.pawelczak@fujitsu.com",
- "url": "https://fujitsu.com"
+ "name": "Please add yourself to the list of contributors",
+ "email": "your.name@example.com",
+ "url": "https://example.com"
  },
  {
  "name": "Michal Bajer",
  "email": "michal.bajer@fujitsu.com",
- "url": "https://fujitsu.com"
- },
- {
- "name": "Eryk Baranowski",
- "email": "Eryk.Baranowski@fujitsu.com",
- "url": "https://fujitsu.com"
+ "url": "https://www.fujitsu.com/global/"
  },
  {
  "name": "Tomasz Awramski",
  "email": "tomasz.awramski@fujitsu.com",
- "url": "https://fujitsu.com"
+ "url": "https://www.fujitsu.com/global/"
  }
  ],
  "main": "dist/lib/main/typescript/index.js",
  "module": "dist/lib/main/typescript/index.js",
- "browser": "dist/cactus-plugin-persistence-fabric.web.umd.js",
  "types": "dist/lib/main/typescript/index.d.ts",
  "files": [
  "dist/*"
  ],
  "scripts": {
- "build": "npm run build-ts && npm run build:dev:backend:postbuild",
+ "build": "npm run build-ts",
  "build-ts": "tsc",
- "build:dev:backend:postbuild": "npm run copy-yarn-lock",
  "codegen": "run-p 'codegen:*'",
  "codegen:openapi": "npm run generate-sdk",
- "copy-yarn-lock": "cp -af ../../yarn.lock ./dist/yarn.lock",
- "generate-sdk": "openapi-generator-cli generate -i ./src/main/json/openapi.json -g typescript-axios -o ./src/main/typescript/generated/openapi/typescript-axios/ --reserved-words-mappings protected=protected --ignore-file-override ../../openapi-generator-ignore",
- "watch": "npm-watch",
- "webpack": "npm-run-all webpack:dev",
- "webpack:dev": "npm-run-all webpack:dev:node webpack:dev:web",
- "webpack:dev:node": "webpack --env=dev --target=node --config ../../webpack.config.js",
- "webpack:dev:web": "webpack --env=dev --target=web --config ../../webpack.config.js"
+ "generate-sdk": "openapi-generator-cli generate -i ./src/main/json/openapi.json -g typescript-axios -o ./src/main/typescript/generated/openapi/typescript-axios/ --reserved-words-mappings protected=protected --ignore-file-override ../../openapi-generator-ignore"
  },
  "dependencies": {
  "@hyperledger/cactus-common": "2.0.0-rc.2",
  "@hyperledger/cactus-core": "2.0.0-rc.2",
  "@hyperledger/cactus-core-api": "2.0.0-rc.2",
- "@hyperledger/cactus-plugin-keychain-memory": "2.0.0-rc.2",
  "@hyperledger/cactus-plugin-ledger-connector-fabric": "2.0.0-rc.2",
+ "async-mutex": "0.4.0",
  "axios": "1.7.2",
- "fabric-protos": "2.2.20",
- "js-sha256": "0.9.0",
  "pg": "8.8.0",
- "safe-stable-stringify": "2.4.3",
- "socket.io": "4.6.2"
+ "run-time-error-cjs": "1.4.0",
+ "uuid": "10.0.0"
  },
  "devDependencies": {
+ "@hyperledger/cactus-plugin-keychain-memory": "2.0.0-rc.2",
  "@hyperledger/cactus-test-tooling": "2.0.0-rc.2",
+ "@openapitools/openapi-generator-cli": "2.7.0",
  "@types/express": "4.17.21",
  "@types/pg": "8.6.5",
- "@types/uuid": "10.0.0",
  "body-parser": "1.20.2",
  "express": "4.19.2",
- "fabric-network": "2.2.20",
- "uuid": "10.0.0"
+ "jest-extended": "4.0.1",
+ "socket.io": "4.6.2"
  },
  "engines": {
  "node": ">=18",
@@ -96,13 +82,5 @@
  "publishConfig": {
  "access": "public"
  },
- "browserMinified": "dist/cactus-plugin-persistence-fabric.web.umd.min.js",
- "mainMinified": "dist/cactus-plugin-persistence-fabric.node.umd.min.js",
- "watch": {
- "codegen:openapi": {
- "patterns": [
- "./src/main/json/openapi.json"
- ]
- }
- }
+ "watch": {}
 }