Feature/environment multi

README.md

@@ -37,7 +37,7 @@ When building the reference implementation, you can [manually build every compon
 
 * [Docker](https://www.docker.com)
 
-If you build with docker [MongoDB V6](https://www.mongodb.com), [NodeJS V16](https://nodejs.org) and [Nats 1.12.2](https://nats.io/) will be installed and configured automatically.
+If you build with docker [MongoDB V6](https://www.mongodb.com), [NodeJS V16](https://nodejs.org) and [Nats 1.12.2](https://nats.io/) will be installed and configured automatically. Make sure you use Docker Compose V2 (comes with Docker Desktop > 3.6.0) as at https://docs.docker.com/compose/install/
 
 ### Installation
 

api-gateway/.env

@@ -1,3 +1,6 @@
+ENV="develop"
+HEDERA_NET="testnet"
+PREUSED_HEDERA_NET="testnet"
 MQ_ADDRESS="localhost"
 SERVICE_CHANNEL="api-gateway"
 MRV_ADDRESS="http://localhost:3003/mrv"

api-gateway/.env.docker

@@ -1,3 +1,5 @@
+HEDERA_NET="testnet"
+PREUSED_HEDERA_NET="testnet"
 MQ_ADDRESS="message-broker"
 SERVICE_CHANNEL="api-gateway"
 MRV_ADDRESS="http://message-broker:3003/mrv"

api-gateway/Dockerfile

@@ -36,7 +36,7 @@ COPY --from=interfacesModuleBuilder /usr/local/interfaces/guardian-interfaces-*.
 COPY --from=commonModuleBuilder /usr/local/common/guardian-common-*.tgz /tmp/common.tgz
 COPY --from=apiGatewayBuilder  /usr/local/api-gateway/yarn.lock ./
 COPY ./api-gateway/package.json ./
-COPY ./api-gateway/.env.docker ./.env
+# COPY ./api-gateway/.env.docker ./.env
 RUN node -e "const fs=require('fs'); const input=JSON.parse(fs.readFileSync('package.json')); input.dependencies['@guardian/interfaces']='file:/tmp/interfaces.tgz'; fs.writeFileSync('package.json', JSON.stringify(input));"
 RUN node -e "const fs=require('fs'); const input=JSON.parse(fs.readFileSync('package.json')); input.dependencies['@guardian/common']='file:/tmp/common.tgz'; fs.writeFileSync('package.json', JSON.stringify(input));"
 RUN yarn install --frozen-lockfile

auth-service/.env

@@ -1,3 +1,6 @@
+ENV="develop"
+HEDERA_NET="testnet"
+PREUSED_HEDERA_NET="testnet"
 MQ_ADDRESS="localhost"
 SERVICE_CHANNEL="auth-service"
 ACCESS_TOKEN_SECRET="youraccesstokensecret"

auth-service/.env.docker

@@ -1,3 +1,5 @@
+HEDERA_NET="testnet"
+PREUSED_HEDERA_NET="testnet"
 MQ_ADDRESS="message-broker"
 SERVICE_CHANNEL="auth-service"
 ACCESS_TOKEN_SECRET="youraccesstokensecret"

auth-service/Dockerfile

@@ -36,7 +36,7 @@ COPY --from=interfacesModuleBuilder /usr/local/interfaces/guardian-interfaces-*.
 COPY --from=commonModuleBuilder /usr/local/common/guardian-common-*.tgz /tmp/common.tgz
 COPY --from=authServiceBuilder  /usr/local/auth-service/yarn.lock ./
 COPY ./auth-service/package.json ./
-COPY ./auth-service/.env.docker ./.env
+# COPY ./auth-service/.env.docker ./.env
 RUN node -e "const fs=require('fs'); const input=JSON.parse(fs.readFileSync('package.json')); input.dependencies['@guardian/interfaces']='file:/tmp/interfaces.tgz'; fs.writeFileSync('package.json', JSON.stringify(input));"
 RUN node -e "const fs=require('fs'); const input=JSON.parse(fs.readFileSync('package.json')); input.dependencies['@guardian/common']='file:/tmp/common.tgz'; fs.writeFileSync('package.json', JSON.stringify(input));"
 RUN yarn install --frozen-lockfile

common/src/helpers/db-helper.ts

@@ -13,7 +13,9 @@ import { DataBaseNamingStrategy } from './db-naming-strategy';
 export const COMMON_CONNECTION_CONFIG: any = {
     type: 'mongo',
     namingStrategy: DataBaseNamingStrategy,
-    dbName: process.env.DB_DATABASE,
+    dbName: (process.env.ENV||(process.env.HEDERA_NET!==process.env.PREUSED_HEDERA_NET))?
+        `${process.env.ENV}_${process.env.HEDERA_NET}_${process.env.DB_DATABASE}`:
+        process.env.DB_DATABASE,
     clientUrl:`mongodb://${process.env.DB_HOST}`,
     entities: [
         'dist/entity/*.js'

docker-compose.yml

@@ -62,33 +62,45 @@ services:
       - ./config:/vault/config:rw
 
   logger-service:
+    env_file:
+      - ./logger-service/.env.docker
     build:
       context: .
       dockerfile: ./logger-service/Dockerfile
     depends_on:
       - message-broker
-
+    environment:
+      - ENV=${GUARDIAN_ENV}
+
   worker-service-1:
+    env_file:
+      - ./worker-service/.env.docker
     build:
       context: .
       dockerfile: ./worker-service/Dockerfile
     depends_on:
       - ipfs-node
       - auth-service
     environment:
-      SERVICE_CHANNEL: worker.1
-
+      - SERVICE_CHANNEL="worker.1"
+      - ENV=${GUARDIAN_ENV}
+
   worker-service-2:
+    env_file:
+      - ./worker-service/.env.docker
     build:
       context: .
       dockerfile: ./worker-service/Dockerfile
     depends_on:
       - ipfs-node
       - auth-service
     environment:
-      SERVICE_CHANNEL: worker.2
-
+      - SERVICE_CHANNEL="worker.2"
+      - ENV=${GUARDIAN_ENV}
+
   auth-service:
+    env_file:
+      - ./auth-service/.env.docker
     build:
       context: .
       dockerfile: ./auth-service/Dockerfile
@@ -97,8 +109,12 @@ services:
       - vault
       - message-broker
       - logger-service
-
+    environment:
+      - ENV=${GUARDIAN_ENV}
+
   api-gateway:
+    env_file:
+      - ./api-gateway/.env.docker
     build:
       context: .
       dockerfile: ./api-gateway/Dockerfile
@@ -110,8 +126,12 @@ services:
       - guardian-service
       - auth-service
       - logger-service
-
+    environment:
+      - ENV=${GUARDIAN_ENV}
+
   policy-service:
+    env_file:
+      - ./policy-service/.env.docker
     build:
       context: .
       dockerfile: ./policy-service/Dockerfile
@@ -120,8 +140,12 @@ services:
       - message-broker
       - auth-service
       - logger-service
-
+    environment:
+      - ENV=${GUARDIAN_ENV}
+
   guardian-service:
+    env_file:
+      - ./guardian-service/.env.docker
     build:
       context: .
       dockerfile: ./guardian-service/Dockerfile
@@ -133,7 +157,9 @@ services:
       - worker-service-1
       - worker-service-2
       - policy-service
-
+    environment:
+      - ENV=${GUARDIAN_ENV}
+
   mrv-sender:
     build:
       context: .

docs/environments/data-separation-concerns.md

@@ -0,0 +1,24 @@
+
+### Data level separation of concerns
+###### 1694 
+In order to discriminate data stored to the database it has been introduced a different database name per each environment. 
+instead of working on the collection names or introducing new fields in the collection it has been preferred to implement the solution changing the whole database name. In this way different data for the different environments will have his own database.
+
+The new db names are going to have the following format  \<environment\>_db_name.
+
+To the purpose of define the Environment name in a global and univocal manner it as been introduced a global .env file at the level of the root guardian folder next to the docker_compose file in a way that docker compose has the visibility  to load the new GUARDIAN_ENV.
+```
+.env example:
+	 GUARDIAN_ENV=prod
+```
+Therefore beside the environment variables loaded in the docker compose file using the env_file, as in 1604, it has been introduced the usage of “environment” specification to load the new GUARDIAN_ENV variable at docker-compose level. The usage of this environment variable has allowed to parametrize the database name in the common library: db-helper.ts 
+
+Hence the user will be able to use the same infrastructure and maintain their production Guardian environment data  completely separated from user other environments data  as test, preview or demo environment. 
+
+The switch  through the different environments is allowed by the configuration of just one parameter in the .env file. 
+
+If the GUARDIAN_ENV parameter is left blank the system will keep behaving in the same way as now and the original database names will be used for the data. 
+
+Therefore the modification will not impact immediately but new logical environments usage are allowed to share the same infrastructure from now on without data environment separation concerns
+
+Make sure you use Docker Compose V2 (comes with Docker Desktop > 3.6.0) as at https://docs.docker.com/compose/install/

docs/environments/indipendent-services-images.md

@@ -0,0 +1,9 @@
+
+### Indipendent services images
+###### 1604 
+The dockerized services images needs to be indipendent from the environment that describes the context in wich the images itself are running.
+
+In each service loading .env files at build time by means of dockerfiles, forces rebuilding the docker image for changes to be applied to the environment. 
+
+To prevent this behavior the usage of “env_file” has been Introduced in the docker_compose file. In this way the environment variables are loaded in each container during the bootstrap of the application and passed to the image without the need to rebuild the image itself.
+The dockerfiles have been changed accordingly: the command “copy” of the .env file was commented out. Actually the .env file is not needed at build time while it’s going to be charged during the bootstrap of the containers at the compose-level.

docs/environments/persistance-configuration-according-environment.md

@@ -0,0 +1,20 @@
+###  Multi session consistency according to environment
+###### 1888, 1696.
+
+Content of data stored during Guardian operative sessions are discriminated according to the environment so that they are always consistent with the data persisted in Hedera Network. This is guaranteed for every operative data and messages exchanged between services that are persisted during different sessions. With the word “session” is intended the time in which the Guardian platform is up and running with the same environment between two different start and stop of the Guardian system.
+Persisted data during each session regarding  transactions both towards Headera net and or Guardian database are easily discriminated in terms of environment  and remain consistent with target Hedera network. 
+
+The persistence consistency is guaranteed leveraging the environment variable which describe the target Headera Network(HEDERA_NET) used  together with the "Guardian ecosystem-environment" name itself (GUARDIAN_ENV). 
+This two environment attribute can be considered as a key of the other session parameters, infact there is a functional dependency between the couple  <GuardianEnvironment, HederaNetwork> and the data written to the DB during a working session.
+
+The implementation goes in the same line as Data level separation of concerns: in order to discriminate data stored to the database it has been added a different database per each Hedera network and Guardian environment. The new db names have the following format  
+
+	<GUARDIAN_ENV>_<HEDERA_NET>_db_name.
+
+![persistence](https://user-images.githubusercontent.com/70752752/226858513-eb732e4e-ca6e-4263-aca7-3428568186d7.png)
+
+It has been introduced a new parameter PREUSED_HEDERA_NET, this parameter is intended to hold the target Hedera network that the system already started to notarized data to. The PREUSED_HEDERA_NET can assume the values mainnet, testnet, previewnet, localnode.
+
+To mantain the usage of the current databse the GUARDIAN_ENV parameter has to be left empty while the PREUSED_HEDERA_NET should be configured as stated before. 
+
+Using this configuration the system will keep behaving in the same way as now and the original database names will be used for the data related to the currently used HEDERA_NET and current Guardian environment. In this way the modification will not impact the current data but will be possible to define multiple different environments and hedera net target BC sharing the same infrastructure without data separation concerns.

docs/getting-started/getting-started/installation.md

@@ -63,7 +63,7 @@ IPFS_PROVIDER="web3storage"
 
 To generate Web3.Storage API KEY. Please follow the steps from [https://web3.storage/docs/#quickstart](https://web3.storage/docs/#quickstart) to obtain it. To know complete information on generating API Key please check : [how-to-generate-web3.storage-api-key.md](how-to-generate-web3.storage-api-key.md "mention")
 
-4\. Build and launch with Docker. Please note that this build is meant to be used in production and will not contain any debug information. From the project's root folder:
+4\. Build and launch with Docker. Make sure you use Docker Compose V2 (comes with Docker Desktop > 3.6.0) as at https://docs.docker.com/compose/install/. Please note that this build is meant to be used in production and will not contain any debug information. From the project's root folder:
 
 ```
 docker-compose up -d --build

guardian-service/.env

@@ -1,3 +1,6 @@
+ENV="develop"
+HEDERA_NET="testnet"
+PREUSED_HEDERA_NET="testnet"
 MQ_ADDRESS="localhost"
 SERVICE_CHANNEL="guardian.1"
 DB_HOST="localhost"
@@ -7,7 +10,6 @@ INITIAL_STANDARD_REGISTRY_BALANCE="100"
 OPERATOR_ID="..."
 OPERATOR_KEY="..."
 INITIALIZATION_TOPIC_ID="0.0.2030"
-HEDERA_NET="testnet"
 MESSAGE_LANG="en-US"
 LOG_LEVEL="1"
 SEND_KEYS_TO_VAULT="TRUE"

guardian-service/.env.docker

@@ -1,3 +1,5 @@
+HEDERA_NET="testnet"
+PREUSED_HEDERA_NET="testnet"
 MQ_ADDRESS="message-broker"
 SERVICE_CHANNEL="guardian.1"
 DB_HOST="mongo"
@@ -7,7 +9,6 @@ INITIAL_STANDARD_REGISTRY_BALANCE="100"
 OPERATOR_ID="..."
 OPERATOR_KEY="..."
 INITIALIZATION_TOPIC_ID="0.0.2030"
-HEDERA_NET="testnet"
 MESSAGE_LANG="en-US"
 LOG_LEVEL="1"
 SEND_KEYS_TO_VAULT="TRUE"

guardian-service/Dockerfile

@@ -36,7 +36,7 @@ COPY --from=interfacesModuleBuilder /usr/local/interfaces/guardian-interfaces-*.
 COPY --from=commonModuleBuilder /usr/local/common/guardian-common-*.tgz /tmp/common.tgz
 COPY --from=guardianServiceBuilder  /usr/local/guardian-service/yarn.lock ./
 COPY ./guardian-service/package.json ./
-COPY ./guardian-service/.env.docker ./.env
+# COPY ./guardian-service/.env.docker ./.env
 COPY ./guardian-service/system-schemas ./system-schemas/.
 RUN node -e "const fs=require('fs'); const input=JSON.parse(fs.readFileSync('package.json')); input.dependencies['@guardian/interfaces']='file:/tmp/interfaces.tgz'; fs.writeFileSync('package.json', JSON.stringify(input));"
 RUN node -e "const fs=require('fs'); const input=JSON.parse(fs.readFileSync('package.json')); input.dependencies['@guardian/common']='file:/tmp/common.tgz'; fs.writeFileSync('package.json', JSON.stringify(input));"

logger-service/.env

@@ -1,3 +1,6 @@
+ENV="develop"
+HEDERA_NET="testnet"
+PREUSED_HEDERA_NET="testnet"
 MQ_ADDRESS="localhost"
 SERVICE_CHANNEL="logger-service"
 DB_HOST="localhost"

logger-service/.env.docker

@@ -1,3 +1,5 @@
+HEDERA_NET="testnet"
+PREUSED_HEDERA_NET="testnet"
 MQ_ADDRESS="message-broker"
 SERVICE_CHANNEL="logger-service"
 DB_HOST="mongo"

logger-service/Dockerfile

@@ -36,7 +36,7 @@ COPY --from=interfacesModuleBuilder /usr/local/interfaces/guardian-interfaces-*.
 COPY --from=commonModuleBuilder /usr/local/common/guardian-common-*.tgz /tmp/common.tgz
 COPY --from=loggerServiceBuilder  /usr/local/logger-service/yarn.lock ./
 COPY ./logger-service/package.json ./
-COPY ./logger-service/.env.docker ./.env
+# COPY ./logger-service/.env.docker ./.env
 RUN node -e "const fs=require('fs'); const input=JSON.parse(fs.readFileSync('package.json')); input.dependencies['@guardian/interfaces']='file:/tmp/interfaces.tgz'; fs.writeFileSync('package.json', JSON.stringify(input));"
 RUN node -e "const fs=require('fs'); const input=JSON.parse(fs.readFileSync('package.json')); input.dependencies['@guardian/common']='file:/tmp/common.tgz'; fs.writeFileSync('package.json', JSON.stringify(input));"
 RUN yarn install --frozen-lockfile

policy-service/.env

@@ -1,8 +1,10 @@
+ENV="develop"
+HEDERA_NET="testnet"
+PREUSED_HEDERA_NET="testnet"
 MQ_ADDRESS="localhost"
 SERVICE_CHANNEL="policy-service"
 DB_HOST="localhost"
 DB_DATABASE="guardian_db"
-HEDERA_NET="testnet"
 MESSAGE_LANG="en-US"
 LOG_LEVEL="1"
 BBS_SIGNATURES_MODE="WASM"

policy-service/.env.docker

@@ -1,8 +1,9 @@
+HEDERA_NET="testnet"
+PREUSED_HEDERA_NET="testnet"
 MQ_ADDRESS="message-broker"
 SERVICE_CHANNEL="policy-service"
 DB_HOST="mongo"
 DB_DATABASE="guardian_db"
-HEDERA_NET="testnet"
 MESSAGE_LANG="en-US"
 LOG_LEVEL="1"
 BBS_SIGNATURES_MODE="WASM"

policy-service/Dockerfile

@@ -36,7 +36,7 @@ COPY --from=interfacesModuleBuilder /usr/local/interfaces/guardian-interfaces-*.
 COPY --from=commonModuleBuilder /usr/local/common/guardian-common-*.tgz /tmp/common.tgz
 COPY --from=policyServiceBuilder  /usr/local/policy-service/yarn.lock ./
 COPY ./policy-service/package.json ./
-COPY ./policy-service/.env.docker ./.env
+# COPY ./policy-service/.env.docker ./.env
 RUN node -e "const fs=require('fs'); const input=JSON.parse(fs.readFileSync('package.json')); input.dependencies['@guardian/interfaces']='file:/tmp/interfaces.tgz'; fs.writeFileSync('package.json', JSON.stringify(input));"
 RUN node -e "const fs=require('fs'); const input=JSON.parse(fs.readFileSync('package.json')); input.dependencies['@guardian/common']='file:/tmp/common.tgz'; fs.writeFileSync('package.json', JSON.stringify(input));"
 RUN yarn install --frozen-lockfile

worker-service/.env

@@ -1,3 +1,6 @@
+ENV="develop"
+HEDERA_NET="testnet"
+PREUSED_HEDERA_NET="testnet"
 MQ_ADDRESS="localhost"
 SERVICE_CHANNEL="worker.1"
 MIN_PRIORITY="0"

worker-service/.env.docker

@@ -1,3 +1,5 @@
+HEDERA_NET="testnet"
+PREUSED_HEDERA_NET="testnet"
 MQ_ADDRESS="message-broker"
 SERVICE_CHANNEL="worker.1"
 MIN_PRIORITY="0"
@@ -6,7 +8,7 @@ TASK_TIMEOUT="300"
 REFRESH_INTERVAL="60"
 IPFS_TIMEOUT="720"
 IPFS_PROVIDER="web3storage" # 'web3storage' or 'local'
-IPFS_PUBLIC_GATEWAY="https://ipfs.io/ipfs/${cid}"
+IPFS_PUBLIC_GATEWAY='https://ipfs.io/ipfs/${cid}'
 IPFS_STORAGE_API_KEY="..."
 IPFS_NODE_ADDRESS="http://ipfs-node:5002"
 MAX_HEDERA_TIMEOUT="600"

worker-service/Dockerfile

@@ -36,7 +36,7 @@ COPY --from=interfacesModuleBuilder /usr/local/interfaces/guardian-interfaces-*.
 COPY --from=commonModuleBuilder /usr/local/common/guardian-common-*.tgz /tmp/common.tgz
 COPY --from=workerServiceBuilder  /usr/local/worker-service/yarn.lock ./
 COPY ./worker-service/package.json ./
-COPY ./worker-service/.env.docker ./.env
+# COPY ./worker-service/.env.docker ./.env
 RUN node -e "const fs=require('fs'); const input=JSON.parse(fs.readFileSync('package.json')); input.dependencies['@guardian/interfaces']='file:/tmp/interfaces.tgz'; fs.writeFileSync('package.json', JSON.stringify(input));"
 RUN node -e "const fs=require('fs'); const input=JSON.parse(fs.readFileSync('package.json')); input.dependencies['@guardian/common']='file:/tmp/common.tgz'; fs.writeFileSync('package.json', JSON.stringify(input));"
 RUN yarn install --frozen-lockfile

yarn.lock

@@ -1,5 +1,3 @@
-# This file is generated by running "yarn install" inside your project.
-# Manual changes might be lost - proceed with caution!
 
 __metadata:
   version: 6