Merge pull request #334 from jembi/create-additional-config-for-notif…

…ication-diplay-fields Create additional config for notification display fields and refactor
jembi · Oct 15, 2024 · 77380ae · 77380ae
2 parents fc2ac46 + af276c0
commit 77380ae
Show file tree

Hide file tree

Showing 12 changed files with 93 additions and 64 deletions.
diff --git a/JeMPI_Apps/JeMPI_AsyncReceiver/src/main/java/org/jembi/jempi/async_receiver/Main.java b/JeMPI_Apps/JeMPI_AsyncReceiver/src/main/java/org/jembi/jempi/async_receiver/Main.java
@@ -15,6 +15,7 @@
 import org.jembi.jempi.shared.serdes.JsonPojoSerializer;
 import org.jembi.jempi.shared.utils.AppUtils;
 import org.apache.commons.codec.language.Soundex;
+import org.jetbrains.annotations.NotNull;
 
 import java.io.IOException;
 import java.nio.file.*;
@@ -153,30 +154,7 @@ private void apacheReadCSV(
 
             for (CSVRecord csvRecord : csvParser) {
                final var patientRecord = demographicData(csvRecord);
-               String givenNameField = FIELDS_CONFIG.nameValidationFields.get(0);
-               String familyNameField = FIELDS_CONFIG.nameValidationFields.get(1);
-               String givenName = patientRecord.fields.stream()
-                  .filter(field -> givenNameField.equals(field.ccTag()))
-                  .map(DemographicData.DemographicField::value)
-                  .findFirst()
-                  .orElse("");
-               String familyName = patientRecord.fields.stream()
-                  .filter(field -> familyNameField.equals(field.ccTag()))
-                  .map(DemographicData.DemographicField::value)
-                  .findFirst()
-                  .orElse("");
-
-               String partitionKey = "";
-               if (!givenName.isEmpty()) {
-                  partitionKey += new Soundex().soundex(givenName);
-               }
-               if (!familyName.isEmpty()) {
-                  partitionKey += new Soundex().soundex(familyName);
-               }
-               if (givenName.isEmpty() && familyName.isEmpty()) {
-                  partitionKey += "Unknown";
-               }
-               LOGGER.info("Using Kafka topic/partition: " + partitionKey);
+               String partitionKey = generateKafkaPartitionKey(patientRecord);
 
                final var interactionEnvelop = new InteractionEnvelop(InteractionEnvelop.ContentType.BATCH_INTERACTION,
                                                                      tag,
@@ -205,6 +183,30 @@ private void apacheReadCSV(
       }
    }
 
+   private static @NotNull String generateKafkaPartitionKey(final DemographicData patientRecord) {
+      StringBuilder partitionKey = new StringBuilder();
+
+      for (String field : FIELDS_CONFIG.fieldsForKafkaKeyGen) {
+         String value = patientRecord.fields.stream()
+                                            .filter(fieldData -> field.equals(fieldData.ccTag()))
+                                            .map(DemographicData.DemographicField::value)
+                                            .findFirst()
+                                            .orElse("");
+
+         if (!value.isEmpty()) {
+            partitionKey.append(new Soundex().soundex(value));
+         }
+      }
+
+      if (partitionKey.length() == 0) {
+         partitionKey.append("Unknown");
+      }
+
+      String partitionKeyStr = partitionKey.toString();
+      LOGGER.info("Using Kafka topic/partition: {}", partitionKeyStr);
+      return partitionKeyStr;
+   }
+
    private String updateStan(
          final String stanDate,
          final int recCount) {

diff --git a/JeMPI_Apps/JeMPI_EM_Scala/src/main/scala/org/jembi/jempi/em/config/Config.scala b/JeMPI_Apps/JeMPI_EM_Scala/src/main/scala/org/jembi/jempi/em/config/Config.scala
@@ -61,10 +61,11 @@ case class Rules(
 )
 
 case class Config(
-    auxInteractionFields: Option[Array[auxField]],
-    auxGoldenRecordFields: Option[Array[auxField]],
-    additionalNodes: Option[Array[AdditionalNode]],
-    demographicFields: Array[DemographicField],
-    rules: Rules,
-    nameValidationFields: Option[List[String]]
+                   auxInteractionFields: Option[Array[auxField]],
+                   auxGoldenRecordFields: Option[Array[auxField]],
+                   additionalNodes: Option[Array[AdditionalNode]],
+                   demographicFields: Array[DemographicField],
+                   rules: Rules,
+                   fieldsForKafkaKeyGen: Option[List[String]],
+                   nameFieldsForNotificationDisplay: Option[List[String]]
 )
diff --git a/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/config/FieldsConfig.java b/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/config/FieldsConfig.java
@@ -31,7 +31,8 @@ public final class FieldsConfig {
    public final List<AuxField> userAuxInteractionFields;
    public final List<AuxField> userAuxGoldenRecordFields;
    public final List<DemographicField> demographicFields;
-   public final List<String> nameValidationFields;
+   public final List<String> fieldsForKafkaKeyGen;
+   public final List<String> nameFieldsForNotificationDisplay;
    public final List<AdditionalNode> additionalNodes;
 
    public FieldsConfig(final JsonConfig jsonConfig) {
@@ -89,7 +90,13 @@ public FieldsConfig(final JsonConfig jsonConfig) {
                        );
       optionalInteractionAuxIdIdx = auxInteractionAuxIdIdx[0];
 
-      nameValidationFields = jsonConfig.nameValidationFields().stream().toList();
+      fieldsForKafkaKeyGen = jsonConfig.fieldsForKafkaKeyGen() != null
+            ? List.copyOf(jsonConfig.fieldsForKafkaKeyGen())
+            : List.of();
+
+      nameFieldsForNotificationDisplay = jsonConfig.nameFieldsForNotificationDisplay() != null
+            ? List.copyOf(jsonConfig.nameFieldsForNotificationDisplay())
+            : List.of();
 
       demographicFields = jsonConfig.demographicFields()
                                     .stream()

diff --git a/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/config/input/JsonConfig.java b/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/config/input/JsonConfig.java
@@ -19,7 +19,8 @@ public record JsonConfig(
       @JsonProperty("auxGoldenRecordFields") List<AuxGoldenRecordField> auxGoldenRecordFields,
       List<AdditionalNode> additionalNodes,
       List<DemographicField> demographicFields,
-      List<String> nameValidationFields,
+      @JsonProperty("fieldsForKafkaKeyGen") List<String> fieldsForKafkaKeyGen,
+      @JsonProperty("nameFieldsForNotificationDisplay") List<String> nameFieldsForNotificationDisplay,
       Rules rules) {
 
    private static final Logger LOGGER = LogManager.getLogger(JsonConfig.class);

diff --git a/...ed/config/input/NameValidationFields.java → ...onfig/input/NameFieldsForKafkaKeyGen.java b/...ed/config/input/NameValidationFields.java → ...onfig/input/NameFieldsForKafkaKeyGen.java
@@ -4,7 +4,7 @@
 import java.util.List;
 
 @JsonInclude(JsonInclude.Include.NON_NULL)
-   public record NameValidationFields(
-         List<String> nameValidationFields
+   public record NameFieldsForKafkaKeyGen(
+         List<String> nameFieldsForKafkaKeyGen
          ) {
    }
diff --git a/..._Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/models/ConfigurationModel.java b/..._Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/models/ConfigurationModel.java
@@ -16,7 +16,8 @@ public record Configuration(
                         List<Field> auxGoldenRecordFields,
                         List<Node> additionalNodes,
                         List<DemographicField> demographicFields,
-                        List<String> nameValidationFields,
+                        List<String> fieldsForKafkaKeyGen,
+                        List<String> nameFieldsForNotificationDisplay,
                         Rules rules) {
         }
 

diff --git a/JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/linker/backend/LinkerDWH.java b/JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/linker/backend/LinkerDWH.java
@@ -95,19 +95,16 @@ private static boolean helperUpdateGoldenRecordField(
 
    private static String patientName(final Interaction interaction) {
       var patientRecord = interaction.demographicData();
-      String givenNameField = FIELDS_CONFIG.nameValidationFields.get(0);
-      String familyNameField = FIELDS_CONFIG.nameValidationFields.get(1);
-      String givenName = patientRecord.fields.stream()
-         .filter(field -> givenNameField.equals(field.ccTag()))
-         .map(DemographicData.DemographicField::value)
-         .findFirst()
-         .orElse("");
-      String familyName = patientRecord.fields.stream()
-         .filter(field -> familyNameField.equals(field.ccTag()))
-         .map(DemographicData.DemographicField::value)
-         .findFirst()
-         .orElse("");
-         return (givenName + " " + familyName).trim();
+      String patientDisplayName = FIELDS_CONFIG.nameFieldsForNotificationDisplay.stream()
+                                                                      .map(fieldName -> patientRecord.fields.stream()
+                                                                                                            .filter(field -> fieldName.equals(
+                                                                                                                  field.ccTag()))
+                                                                                                            .map(DemographicData.DemographicField::value)
+                                                                                                            .findFirst()
+                                                                                                            .orElse(""))
+                                                                      .collect(Collectors.joining(" "))
+                                                                      .trim();
+      return patientDisplayName;
    }
 
    /**

diff --git a/devops/linux/docker/data-config/config-reference-auto-generate-fields.json b/devops/linux/docker/data-config/config-reference-auto-generate-fields.json
@@ -135,9 +135,13 @@
       "indexGoldenRecord": "@index(exact)"
     }
   ],
-  "nameValidationFields": [
-            "given_name",
-            "family_name"
+  "fieldsForKafkaKeyGen": [
+    "given_name",
+    "family_name"
+  ],
+  "nameFieldsForNotificationDisplay": [
+    "given_name",
+    "family_name"
   ],
   "rules": {
     "link": {

diff --git a/devops/linux/docker/data-config/config-reference-link-d-validate-dp-match-dp.json b/devops/linux/docker/data-config/config-reference-link-d-validate-dp-match-dp.json
@@ -222,9 +222,13 @@
       "matchMetaData": null
     }
   ],
-  "nameValidationFields": [
-            "given_name",
-            "family_name"
+  "fieldsForKafkaKeyGen": [
+    "given_name",
+    "family_name"
+  ],
+  "nameFieldsForNotificationDisplay": [
+    "given_name",
+    "family_name"
   ],
   "rules": {
     "link": {

diff --git a/devops/linux/docker/data-config/config-reference-link-d.json b/devops/linux/docker/data-config/config-reference-link-d.json
@@ -113,9 +113,13 @@
       "indexGoldenRecord": "@index(exact)"
     }
   ],
-  "nameValidationFields": [
-            "given_name",
-            "family_name"
+  "fieldsForKafkaKeyGen": [
+    "given_name",
+    "family_name"
+  ],
+  "nameFieldsForNotificationDisplay": [
+    "given_name",
+    "family_name"
   ],
   "rules": {
     "link": {

diff --git a/devops/linux/docker/data-config/config-reference-link-dp-minimal.json b/devops/linux/docker/data-config/config-reference-link-dp-minimal.json
@@ -189,9 +189,13 @@
       }
     }
   ],
-  "nameValidationFields": [
-            "given_name",
-            "family_name"
+  "fieldsForKafkaKeyGen": [
+    "given_name",
+    "family_name"
+  ],
+  "nameFieldsForNotificationDisplay": [
+    "given_name",
+    "family_name"
   ],
   "rules": {
     "link": {

diff --git a/devops/linux/docker/data-config/config-reference-link-dp.json b/devops/linux/docker/data-config/config-reference-link-dp.json
@@ -174,9 +174,13 @@
       }
     }
   ],
-  "nameValidationFields": [
-            "given_name",
-            "family_name"
+  "fieldsForKafkaKeyGen": [
+    "given_name",
+    "family_name"
+  ],
+  "nameFieldsForNotificationDisplay": [
+    "given_name",
+    "family_name"
   ],
   "rules": {
     "link": {