Merge branch 'master' into BFD-3685

apps/bfd-data-fda/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.cms.bfd</groupId>
         <artifactId>bfd-parent</artifactId>
-        <version>2.178.0-SNAPSHOT</version>
+        <version>2.179.0-SNAPSHOT</version>
     </parent>
 
     <groupId>gov.cms.bfd.data.fda.utility</groupId>

apps/bfd-data-npi/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>gov.cms.bfd</groupId>
         <artifactId>bfd-parent</artifactId>
-        <version>2.178.0-SNAPSHOT</version>
+        <version>2.179.0-SNAPSHOT</version>
     </parent>
 
     <groupId>gov.cms.bfd.data.npi</groupId>
@@ -70,6 +70,14 @@
             <groupId>com.google.guava</groupId>
             <artifactId>guava</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+        </dependency>
     </dependencies>
     <build>
         <finalName>bfd-data-npi</finalName>

apps/bfd-data-npi/src/main/java/gov/cms/bfd/data/npi/dto/NPIData.java

@@ -0,0 +1,52 @@
+package gov.cms.bfd.data.npi.dto;
+
+import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+
+/** DTO used to supply the server with NPI enrichment information. */
+@Builder
+@Getter
+@AllArgsConstructor
+@NoArgsConstructor
+@JsonInclude(NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class NPIData {
+  /** Provider or Org npi. */
+  String npi;
+
+  /** Entity Type. Will be 1 for Provider, or 2 for Organization. */
+  String entityTypeCode;
+
+  /** Organization name. */
+  String providerOrganizationName;
+
+  /** Taxonomy code. */
+  String taxonomyCode;
+
+  /** Taxonomy display. */
+  String taxonomyDisplay;
+
+  /** Provider name prefix. */
+  String providerNamePrefix;
+
+  /** Provider first name. */
+  String providerFirstName;
+
+  /** Provider middle name. */
+  String providerMiddleName;
+
+  /** Provider last name. */
+  String providerLastName;
+
+  /** Provider suffix. */
+  String providerNameSuffix;
+
+  /** Provider credential. */
+  String providerCredential;
+}

apps/bfd-data-npi/src/main/java/gov/cms/bfd/data/npi/lookup/NPIOrgLookup.java

@@ -1,13 +1,20 @@
 package gov.cms.bfd.data.npi.lookup;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import gov.cms.bfd.data.npi.dto.NPIData;
 import gov.cms.bfd.data.npi.utility.App;
+import java.io.BufferedInputStream;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
+import java.util.zip.InflaterInputStream;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -21,6 +28,9 @@ public class NPIOrgLookup {
   /** A field to return the production org lookup. */
   private static NPIOrgLookup npiOrgLookupForProduction;
 
+  /** Zlib compression header. */
+  private static final byte[] COMPRESSED_HEADER = {120, -100};
+
   /**
    * Factory method for creating a {@link NPIOrgLookup } for production that does not include the
    * fake org name.
@@ -53,7 +63,7 @@ public NPIOrgLookup(Map<String, String> npiOrgMap) {
    * @throws IOException if there is an issue reading file
    */
   public NPIOrgLookup(InputStream npiDataStream) throws IOException {
-    npiOrgHashMap = readNPIOrgDataStream(npiDataStream);
+    npiOrgHashMap = readNPIOrgDataStream(new BufferedInputStream(npiDataStream));
   }
 
   /**
@@ -63,7 +73,7 @@ public NPIOrgLookup(InputStream npiDataStream) throws IOException {
    * @param npiNumber - npiNumber value in claim records
    * @return the npi org data display string
    */
-  public Optional<String> retrieveNPIOrgDisplay(Optional<String> npiNumber) {
+  public Optional<NPIData> retrieveNPIOrgDisplay(Optional<String> npiNumber) {
     /*
      * Handle bad data (e.g. our random test data) if npiNumber is empty
      */
@@ -72,8 +82,14 @@ public Optional<String> retrieveNPIOrgDisplay(Optional<String> npiNumber) {
     }
 
     if (npiOrgHashMap.containsKey(npiNumber.get())) {
-      String npiDisplay = npiOrgHashMap.get(npiNumber.get());
-      return Optional.of(npiDisplay);
+      String json = npiOrgHashMap.get(npiNumber.get());
+      ObjectMapper mapper = new ObjectMapper();
+      try {
+        NPIData npiData = mapper.readValue(json, NPIData.class);
+        return Optional.of(npiData);
+      } catch (JsonProcessingException e) {
+        return Optional.empty();
+      }
     }
 
     return Optional.empty();
@@ -88,29 +104,43 @@ public Optional<String> retrieveNPIOrgDisplay(Optional<String> npiNumber) {
    * @return the hashmapped for npis and the npi org names
    */
   protected Map<String, String> readNPIOrgDataStream(InputStream inputStream) throws IOException {
-    Map<String, String> npiProcessedData = new HashMap<String, String>();
-
-    try (final InputStream npiStream = inputStream;
-        final BufferedReader npiReader = new BufferedReader(new InputStreamReader(npiStream))) {
-
-      String line = "";
-      while ((line = npiReader.readLine()) != null) {
-        String npiDataColumns[] = line.split("\t");
-        if (npiDataColumns.length == 2) {
-          npiProcessedData.put(
-              npiDataColumns[0].replace("\"", ""), npiDataColumns[1].replace("\"", ""));
-        } else if (npiDataColumns.length == 3) {
-          // three columns means it's a practitioner taxonomy
-          npiProcessedData.put(
-              npiDataColumns[0].replace("\"", ""),
-              npiDataColumns[1].replace("\"", "") + "\t" + npiDataColumns[2].replace("\"", ""));
-        }
+    Map<String, String> npiProcessedData = new HashMap<>();
+    // if the stream is compressed, we will have to use InflaterInputStream to read it.
+    boolean isCompressedStream = isStreamDeflated(inputStream);
+    String line;
+    try (final InputStream npiStream =
+            isCompressedStream ? new InflaterInputStream(inputStream) : inputStream;
+        final BufferedReader reader = new BufferedReader(new InputStreamReader(npiStream))) {
+      ObjectMapper objectMapper = new ObjectMapper();
+      while ((line = reader.readLine()) != null) {
+        JsonNode rootNode = objectMapper.readTree(line);
+        String npi = rootNode.path("npi").asText();
+        npiProcessedData.put(npi, line);
       }
     }
-
     return npiProcessedData;
   }
 
+  /**
+   * Checks if a stream is deflated. We will read the first two bytes of the stream to compare
+   * against the Zlib header (used by DeflaterOutputStream), then reset the stream back to the
+   * beginning.
+   *
+   * @param inputStream The stream to check
+   * @return true if the stream is deflated
+   * @throws IOException on read error.
+   */
+  public boolean isStreamDeflated(InputStream inputStream) throws IOException {
+    // Mark the current position in the stream.
+    inputStream.mark(2);
+    // Read the first two bytes
+    byte[] bytes = new byte[2];
+    int bytesRead = inputStream.read(bytes);
+    // Reset the stream to the marked position
+    inputStream.reset();
+    return (bytesRead == 2 && Arrays.equals(bytes, COMPRESSED_HEADER));
+  }
+
   /**
    * Returns a inputStream from file name passed in.
    *

apps/bfd-data-npi/src/main/java/gov/cms/bfd/data/npi/utility/DataUtilityCommons.java

@@ -1,16 +1,15 @@
 package gov.cms.bfd.data.npi.utility;
 
-import com.google.common.base.Strings;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import gov.cms.bfd.data.npi.dto.NPIData;
 import java.io.BufferedOutputStream;
 import java.io.BufferedReader;
-import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.FilenameFilter;
 import java.io.IOException;
 import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
 import java.net.URL;
 import java.nio.charset.Charset;
 import java.nio.charset.CharsetDecoder;
@@ -27,6 +26,7 @@
 import java.util.Objects;
 import java.util.Optional;
 import java.util.stream.Stream;
+import java.util.zip.DeflaterOutputStream;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 import org.apache.commons.csv.CSVFormat;
@@ -59,6 +59,30 @@ public class DataUtilityCommons {
   /** Field for Entity Type Code in CSV. */
   public static final String ENTITY_TYPE_CODE_FIELD = "Entity Type Code";
 
+  /** Field for Provider Credential code in CSV. */
+  private static final String PROVIDER_CREDENTIAL_FIELD = "Provider Credential Text";
+
+  /** Field for Provider first name in CSV. */
+  private static final String PROVIDER_FIRST_NAME_FIELD = "Provider First Name";
+
+  /** Field for Provider middle name in CSV. */
+  private static final String PROVIDER_MIDDLE_NAME_FIELD = "Provider Middle Name";
+
+  /** Field for Provider last name in CSV. */
+  private static final String PROVIDER_LAST_NAME_FIELD = "Provider Last Name (Legal Name)";
+
+  /** Field for Provider prefix in CSV. */
+  private static final String PROVIDER_PREFIX_FIELD = "Provider Name Prefix Text";
+
+  /** Field for Provider Suffix in CSV. */
+  private static final String PROVIDER_SUFFIX_FIELD = "Provider Name Suffix Text";
+
+  /** Code for Provider entity type. */
+  public static final String ENTITY_TYPE_CODE_PROVIDER = "1";
+
+  /** Code for Organization entity type. */
+  public static final String ENTITY_TYPE_CODE_ORGANIZATION = "2";
+
   /**
    * Gets the org names from the npi file.
    *
@@ -330,27 +354,42 @@ private static void convertNpiDataFile(Path convertedNpiDataFile, Path originalN
         BufferedReader reader = new BufferedReader(new InputStreamReader(is, inDec));
         FileOutputStream fw =
             new FileOutputStream(convertedNpiDataFile.toFile().getAbsolutePath());
-        BufferedWriter out = new BufferedWriter(new OutputStreamWriter(fw, outEnc))) {
+        DeflaterOutputStream dos = new DeflaterOutputStream(fw); ) {
       CSVParser csvParser =
           new CSVParser(
               reader,
               CSVFormat.DEFAULT.withFirstRecordAsHeader().withIgnoreHeaderCase().withTrim());
+      ObjectMapper objectMapper = new ObjectMapper();
       for (CSVRecord csvRecord : csvParser) {
         String orgName = csvRecord.get(PROVIDER_ORGANIZATION_NAME_FIELD);
         String npi = csvRecord.get(NPI_FIELD);
         String entityTypeCode = csvRecord.get(ENTITY_TYPE_CODE_FIELD);
         String taxonomyCode = csvRecord.get(TAXONOMY_CODE_FIELD);
-        // entity type code 2 is organization
-        if (!Strings.isNullOrEmpty(entityTypeCode)) {
-          if (Integer.parseInt(entityTypeCode) == 2) {
-            out.write(npi + "\t" + orgName);
-            out.newLine();
-          } else if (!Strings.isNullOrEmpty(taxonomyCode)) {
-            out.write(npi + "\t" + taxonomyCode + "\t" + taxonomyMap.get(taxonomyCode));
-            out.newLine();
-          }
-        }
+        String providerFirstName = csvRecord.get(PROVIDER_FIRST_NAME_FIELD);
+        String providerMiddleName = csvRecord.get(PROVIDER_MIDDLE_NAME_FIELD);
+        String providerLastName = csvRecord.get(PROVIDER_LAST_NAME_FIELD);
+        String providerPrefix = csvRecord.get(PROVIDER_PREFIX_FIELD);
+        String providerSuffix = csvRecord.get(PROVIDER_SUFFIX_FIELD);
+        String providerCredential = csvRecord.get(PROVIDER_CREDENTIAL_FIELD);
+        NPIData npiData =
+            NPIData.builder()
+                .npi(npi)
+                .providerOrganizationName(orgName)
+                .entityTypeCode(entityTypeCode)
+                .taxonomyCode(taxonomyCode)
+                .taxonomyDisplay(taxonomyMap.get(taxonomyCode))
+                .providerFirstName(providerFirstName)
+                .providerMiddleName(providerMiddleName)
+                .providerLastName(providerLastName)
+                .providerNamePrefix(providerPrefix)
+                .providerNameSuffix(providerSuffix)
+                .providerCredential(providerCredential)
+                .build();
+        String json = objectMapper.writeValueAsString(npiData);
+        dos.write(json.getBytes());
+        dos.write("\n".getBytes());
       }
+      dos.close();
     }
   }