feat(mojaloop/#3577): add batch handler (#582)

feat(mojaloop/#3577): add batch handler - mojaloop/project#3577
- Added  `BULK_TRANSFER` property to `EVENT_TYPE_ACTION_TOPIC_MAP` for routing `bulk-prepare` messages in `centralledger/chart-handler-transfer-prepare`
- Updated central-ledger version to include version with batch functionality
- Added configs for enabling/disabling batch functionality
- Added chart for batch functionality
- Updated README with instructions on batch functionality

README.md

@@ -24,6 +24,7 @@
   - [Debugging Charts](#debugging-charts)
   - [Helper scripts](#helper-scripts)
   - [Monitoring Mojaloop](#monitoring-mojaloop)
+  - [Batch Processing](#batch-processing)
 
 ## Quick Links
 
@@ -314,6 +315,34 @@ e.g. `helm --namespace mojaloop del dev`
 
 Refer to [Monitoring Documentation](./monitoring/README.md)
 
+## Batch Processing
+
+To enable batch processing in your system, please follow the steps below:
+
+- **1. Add a New Kafka Topic:**
+    Add a new topic named `topic-transfer-position-batch` to your Kafka configuration. If you are using the example-mojaloop-backend for your backend dependencies, this topic is already added to the Kafka provisioning section by default.
+- **2. Update Mojaloop Values File:**
+    In the Mojaloop values file, make sure to enable the batch_processing_enabled flag in the global configuration.
+
+    ```yaml
+    global:
+        batch_processing_enabled: &CL_BATCH_PROCESSING_ENABLED true
+    ```
+
+    Enabling this variable does the following
+
+    **a. Activates Batch Position Handler:** Triggers the activation of the Batch Position Handler.
+
+    **b. Configures Prepare Handler for Kafka Events:** The configuration change also sets up the Prepare Handler to publish Kafka events to the newly designated batch topic.
+
+    ***Note:***
+
+    ***Please note that this configuration change relies on the use of the YAML anchor CL_BATCH_PROCESSING_ENABLED._***
+
+    ***It's essential to consider the context of your deployment. If you have the entire Helm values file as an override, this configuration change will work seamlessly.***
+
+    ***However, if you are using a Helm override file with only a subset of values overridden, ensure that you include all the configuration parameters associated with the CL_BATCH_PROCESSING_ENABLED anchor. Failing to include these parameters might result in unexpected behavior.***
+
 ## Known Issues
 
 1. Snapshot releases will fail if the commit SHA starts with a 0 - https://github.com/helm/helm/issues/7064

bulk-centralledger/chart-handler-bulk-transfer-fulfil/values.yaml

@@ -9,7 +9,7 @@ global: {}
 image:
   registry: "docker.io"
   repository: mojaloop/central-ledger
-  tag: v17.3.2
+  tag: v17.4.0
   ## Specify a imagePullPolicy
   ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
   ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images

bulk-centralledger/chart-handler-bulk-transfer-get/values.yaml

@@ -9,7 +9,7 @@ global: {}
 image:
   registry: "docker.io"
   repository: mojaloop/central-ledger
-  tag: v17.3.2
+  tag: v17.4.0
   ## Specify a imagePullPolicy
   ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
   ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images

bulk-centralledger/chart-handler-bulk-transfer-prepare/values.yaml

@@ -9,7 +9,7 @@ global: {}
 image:
   registry: "docker.io"
   repository: mojaloop/central-ledger
-  tag: v17.3.2
+  tag: v17.4.0
   ## Specify a imagePullPolicy
   ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
   ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images

bulk-centralledger/chart-handler-bulk-transfer-processing/values.yaml

@@ -9,7 +9,7 @@ global: {}
 image:
   registry: "docker.io"
   repository: mojaloop/central-ledger
-  tag: v17.3.2
+  tag: v17.4.0
   ## Specify a imagePullPolicy
   ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
   ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images

bulk-centralledger/values.yaml

@@ -13,7 +13,7 @@ cl-handler-bulk-transfer-prepare:
   image:
     registry: "docker.io"
     repository: mojaloop/central-ledger
-    tag: v17.3.2
+    tag: v17.4.0
     ## Specify a imagePullPolicy
     ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
     ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images
@@ -396,7 +396,7 @@ cl-handler-bulk-transfer-fulfil:
   image:
     registry: "docker.io"
     repository: mojaloop/central-ledger
-    tag: v17.3.2
+    tag: v17.4.0
     ## Specify a imagePullPolicy
     ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
     ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images
@@ -767,7 +767,7 @@ cl-handler-bulk-transfer-processing:
   image:
     registry: "docker.io"
     repository: mojaloop/central-ledger
-    tag: v17.3.2
+    tag: v17.4.0
     ## Specify a imagePullPolicy
     ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
     ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images
@@ -1147,7 +1147,7 @@ cl-handler-bulk-transfer-get:
   image:
     registry: "docker.io"
     repository: mojaloop/central-ledger
-    tag: v17.3.2
+    tag: v17.4.0
     ## Specify a imagePullPolicy
     ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
     ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images

centralledger/Chart.yaml

@@ -1,8 +1,8 @@
 apiVersion: v2
 description: Central-Ledger Helm chart for Kubernetes
 name: centralledger
-version: 14.1.0
-appVersion: "17.3.2"
+version: 14.2.0
+appVersion: "17.4.0"
 home: http://mojaloop.io
 icon: http://mojaloop.io/images/logo.png
 sources:
@@ -18,31 +18,35 @@ maintainers:
     email: steven.oderayi@infitx.com
 dependencies:
   - name: centralledger-service
-    version: 14.1.0
+    version: 14.2.0
     repository: "file://./chart-service"
     condition: centralledger-service.enabled
   - name: centralledger-handler-transfer-prepare
-    version: 14.1.0
+    version: 14.2.0
     repository: "file://./chart-handler-transfer-prepare"
     condition: centralledger-handler-transfer-prepare.enabled
   - name: centralledger-handler-transfer-position
-    version: 14.1.0
+    version: 14.2.0
     repository: "file://./chart-handler-transfer-position"
     condition: centralledger-handler-transfer-position.enabled
+  - name: centralledger-handler-transfer-position-batch
+    version: 15.3.0
+    repository: "file://./chart-handler-transfer-position-batch"
+    condition: centralledger-handler-transfer-position-batch.enabled
   - name: centralledger-handler-transfer-get
-    version: 14.1.0
+    version: 14.2.0
     repository: "file://./chart-handler-transfer-get"
     condition: centralledger-handler-transfer-get.enabled
   - name: centralledger-handler-transfer-fulfil
-    version: 14.1.0
+    version: 14.2.0
     repository: "file://./chart-handler-transfer-fulfil"
     condition: centralledger-handler-transfer-fulfil.enabled
   - name: centralledger-handler-timeout
-    version: 14.1.0
+    version: 14.2.0
     repository: "file://./chart-handler-timeout"
     condition: centralledger-handler-timeout.enabled
   - name: centralledger-handler-admin-transfer
-    version: 14.1.0
+    version: 14.2.0
     repository: "file://./chart-handler-admin-transfer"
     condition: centralledger-handler-transfer-get.enabled
   - name: common

centralledger/chart-handler-admin-transfer/Chart.yaml

@@ -1,8 +1,8 @@
 apiVersion: v2
 description: Central-Ledger Transfer Prepare Handler Helm chart for Kubernetes
 name: centralledger-handler-admin-transfer
-version: 14.1.0
-appVersion: "17.3.2"
+version: 14.2.0
+appVersion: "17.4.0"
 home: http://mojaloop.io
 icon: http://mojaloop.io/images/logo.png
 sources:

centralledger/chart-handler-admin-transfer/values.yaml

@@ -9,7 +9,7 @@ global: {}
 image:
   registry: "docker.io"
   repository: mojaloop/central-ledger
-  tag: v17.3.2
+  tag: v17.4.0
   ## Specify a imagePullPolicy
   ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
   ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images

centralledger/chart-handler-timeout/Chart.yaml

@@ -1,8 +1,8 @@
 apiVersion: v2
 description: Central-Ledger Timeout Handler Helm chart for Kubernetes
 name: centralledger-handler-timeout
-version: 14.1.0
-appVersion: "17.3.2"
+version: 14.2.0
+appVersion: "17.4.0"
 home: http://mojaloop.io
 icon: http://mojaloop.io/images/logo.png
 sources:

centralledger/chart-handler-timeout/values.yaml

@@ -9,7 +9,7 @@ global: {}
 image:
   registry: "docker.io"
   repository: mojaloop/central-ledger
-  tag: v17.3.2
+  tag: v17.4.0
   ## Specify a imagePullPolicy
   ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
   ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images

centralledger/chart-handler-transfer-fulfil/Chart.yaml

@@ -1,8 +1,8 @@
 apiVersion: v2
 description: Central-Ledger Transfer Fulfil Handler Helm chart for Kubernetes
 name: centralledger-handler-transfer-fulfil
-version: 14.1.0
-appVersion: "17.3.2"
+version: 14.2.0
+appVersion: "17.4.0"
 home: http://mojaloop.io
 icon: http://mojaloop.io/images/logo.png
 sources:

centralledger/chart-handler-transfer-fulfil/values.yaml

@@ -9,7 +9,7 @@ global: {}
 image:
   registry: "docker.io"
   repository: mojaloop/central-ledger
-  tag: v17.3.2
+  tag: v17.4.0
   ## Specify a imagePullPolicy
   ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
   ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images

centralledger/chart-handler-transfer-get/Chart.yaml

@@ -1,8 +1,8 @@
 apiVersion: v2
 description: Central-Ledger Get Transfer Handler Helm chart for Kubernetes
 name: centralledger-handler-transfer-get
-version: 14.1.0
-appVersion: "17.3.2"
+version: 14.2.0
+appVersion: "17.4.0"
 home: http://mojaloop.io
 icon: http://mojaloop.io/images/logo.png
 sources:

centralledger/chart-handler-transfer-get/values.yaml

@@ -9,7 +9,7 @@ global: {}
 image:
   registry: "docker.io"
   repository: mojaloop/central-ledger
-  tag: v17.3.2
+  tag: v17.4.0
   ## Specify a imagePullPolicy
   ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
   ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images

centralledger/chart-handler-transfer-position-batch/Chart.yaml

@@ -0,0 +1,22 @@
+apiVersion: v2
+description: "Central-Ledger Transfer Position Batch Handler Helm chart for Kubernetes."
+name: centralledger-handler-transfer-position-batch
+version: 15.3.0
+appVersion: "17.4.0"
+home: http://mojaloop.io
+icon: http://mojaloop.io/images/logo.png
+sources:
+  - https://github.com/mojaloop/mojaloop
+  - https://github.com/mojaloop/helm
+  - https://github.com/mojaloop/central-ledger
+maintainers:
+  - name: Viaya Kumar Guthi
+    email: vijaya.guthi@infitx.com
+  - name: Steven Oderayi
+    email: steven.oderayi@infitx.com
+dependencies:
+  - name: common
+    repository: "https://mojaloop.github.io/charts/repo"
+    version: 2.0.0
+    tags:
+      - moja-common

centralledger/chart-handler-transfer-position-batch/configs/default-sidecar.json

@@ -0,0 +1,50 @@
+{{- $kafkaHost := ( .Values.config.kafka_host | replace "$release_name" .Release.Name ) -}}
+{
+  "PORT": {{ .Values.sidecar.service.internalPort }},
+  "EVENT_LOGGER_GRPC_HOST": "{{ .Values.sidecar.config.event_log_grpc_host }}",
+  "EVENT_LOGGER_GRPC_PORT": {{ .Values.sidecar.config.event_log_grpc_port }},
+  "INSTRUMENTATION": {
+    "METRICS": {
+      "DISABLED": false,
+      "labels": {
+        "eventId": "*"
+      },
+      "config": {
+        "timeout": 5000,
+        "prefix": "moja_",
+        "defaultLabels": {
+          "serviceName": "{{ template "centralledger-handler-transfer-position-batch.name" . }}-event-sidecar"
+        }
+      }
+    }
+  },
+  "KAFKA": {
+    "TOPIC_TEMPLATES": {
+      "GENERAL_TOPIC_TEMPLATE": {
+        "TEMPLATE": "topic-{{"{{"}}functionality{{"}}"}}",
+        "REGEX": "topic-(.*)"
+      }
+    },
+    "PRODUCER": {
+      "EVENT": {
+        "config": {
+          "options": {
+            "messageCharset": "utf8"
+          },
+          "rdkafkaConf": {
+            "metadata.broker.list": "{{ $kafkaHost }}:{{ .Values.config.kafka_port }}",
+            "client.id": "es-prod-event",
+            "event_cb": true,
+            "dr_cb": true,
+            "socket.keepalive.enable": true,
+            "queue.buffering.max.messages": 10000000
+          },
+          "topicConf": {
+            "request.required.acks": "all",
+            "partitioner": "{{ .Values.config.kafka_partitioner }}"
+          }
+        }
+      }
+    }
+  }
+}