Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Dev ground truth #141

Merged
merged 2 commits into from
Nov 6, 2023
Merged

Dev ground truth #141

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
037f388
decouple aux_id
w-hayes Nov 5, 2023
b65aeef
decouple aux_id
w-hayes Nov 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion JeMPI_Apps/JeMPI_Configuration/reference/config-reference.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@
},
{
"fieldName": "aux_id",
"fieldType": "String"
"fieldType": "String",
"source": "aux_id"
}
],
"additionalNodes": [
Expand Down
1 change: 1 addition & 0 deletions JeMPI_Apps/JeMPI_Configuration/src/main/scala/configuration/Config.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ case class UniqueField(fieldName: String,
csvCol: Option[Int],
index: Option[String],
isList: Option[Boolean],
source: Option[String],
default: Option[String])

case class AdditionalNode(nodeName: String,
Expand Down
16 changes: 10 additions & 6 deletions JeMPI_Apps/JeMPI_Configuration/src/main/scala/configuration/CustomAsyncHelper.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,11 @@ private object CustomAsyncHelper {

def additionalNodeFields(additionalNode: AdditionalNode): String =
additionalNode.fields
.map(f => s"""${" " * 3}private static final int ${additionalNode.nodeName.toUpperCase}_${f.fieldName.toUpperCase}_COL_NUM = ${f.csvCol.get}""")
.mkString(
s""";
.map(f => if (f.csvCol.isEmpty) "" else
s""" private static final int ${additionalNode.nodeName.toUpperCase}_${f.fieldName.toUpperCase}_COL_NUM = ${f.csvCol.get};
|""".stripMargin)
.mkString("")
.stripMargin
.stripTrailing()
end additionalNodeFields

Expand All @@ -67,8 +68,8 @@ private object CustomAsyncHelper {
(if (config.uniqueInteractionFields.isEmpty) "" else
uniqueInteractionFields(config.uniqueInteractionFields.get))
+
(if (config.additionalNodes.isEmpty) "" else
config.additionalNodes.get.map(x => s"""${additionalNodeFields(x)};""").mkString(sys.props("line.separator"))) + sys.props("line.separator")
(if (config.additionalNodes.isEmpty || config.additionalNodes.get.isEmpty) "" else
config.additionalNodes.get.map(x => s"""${additionalNodeFields(x)}""").mkString(sys.props("line.separator"))) + sys.props("line.separator")
+
config
.demographicFields
Expand Down Expand Up @@ -112,7 +113,10 @@ private object CustomAsyncHelper {
def customNodeConstructor(additionalNode: AdditionalNode): String =

def arguments(fields: Array[AdditionalNodeField]): String =
fields.map(f => s""" csvRecord.get(${additionalNode.nodeName.toUpperCase}_${f.fieldName.toUpperCase()}_COL_NUM)""").mkString(s",${sys.props("line.separator")}")
fields.map(f => if (f.csvCol.isEmpty)
s"""${" " * 9}null"""
else
s"""${" " * 9}csvRecord.get(${additionalNode.nodeName.toUpperCase}_${f.fieldName.toUpperCase()}_COL_NUM)""").mkString(s",${sys.props("line.separator")}")
end arguments

s""" static Custom${additionalNode.nodeName} custom${additionalNode.nodeName}(final CSVRecord csvRecord) {
Expand Down
21 changes: 21 additions & 0 deletions JeMPI_Apps/JeMPI_Configuration/src/main/scala/configuration/CustomPatient.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,18 @@ private object CustomPatient {
.dropRight(1)
end fields

def fromInteraction(): String =
if (config.uniqueGoldenRecordFields.isEmpty) "" else
config
.uniqueGoldenRecordFields
.get
.map(f => if (f.source.isEmpty) "" else
s""",
|${" " * 9}uniqueInteractionData.${Utils.snakeCaseToCamelCase(f.source.get)}()""".stripMargin)
.mkString(sys.props("line.separator"))
.trim
end fromInteraction

println("Creating " + classCustomUniqueGoldenRecordDataFile)
val file: File = new File(classCustomUniqueGoldenRecordDataFile)
val writer: PrintWriter = new PrintWriter(file)
Expand All @@ -120,8 +132,17 @@ private object CustomPatient {
|
|import com.fasterxml.jackson.annotation.JsonInclude;
|
|import java.time.LocalDateTime;
|
|@JsonInclude(JsonInclude.Include.NON_NULL)
|public record $customClassNameCustomUniqueGoldenRecordData(${fields(config)}) {
|
| public CustomUniqueGoldenRecordData(final CustomUniqueInteractionData uniqueInteractionData) {
| this(LocalDateTime.now(),
| true${fromInteraction()}
| );
| }
|
|}
|""".stripMargin)
writer.flush()
Expand Down
4 changes: 2 additions & 2 deletions JeMPI_Apps/JeMPI_GUI/src/main/java/org/jembi/jempi/gui/Cache.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ private static final class GoldenRecordBuffer {
private static String[] getGoldenRecordVector(final ApiModels.ApiExpandedGoldenRecord expandedGoldenRecord) {
final var demographicFields = CustomDemographicData.class.getDeclaredFields();
final String[] vector = new String[3 + demographicFields.length + 1];
vector[0] = expandedGoldenRecord.goldenRecord().uniqueGoldenRecordData().auxId();
vector[0] = ""; // expandedGoldenRecord.goldenRecord().uniqueGoldenRecordData().auxId();
vector[1] = expandedGoldenRecord.goldenRecord().uid();
vector[2] = expandedGoldenRecord.goldenRecord().uniqueGoldenRecordData().auxDateCreated().toString();
for (int i = 0; i < demographicFields.length; i++) {
Expand All @@ -72,7 +72,7 @@ private static String[] getGoldenRecordVector(final ApiModels.ApiExpandedGoldenR
private static String[] getInteractionVector(final ApiModels.ApiInteractionWithScore interactionWithScore) {
final var demographicFields = CustomDemographicData.class.getDeclaredFields();
final String[] vector = new String[3 + demographicFields.length + 1];
vector[0] = interactionWithScore.interaction().uniqueInteractionData().auxId();
vector[0] = ""; // interactionWithScore.interaction().uniqueInteractionData().auxId();
vector[1] = interactionWithScore.interaction().uid();
vector[2] = interactionWithScore.interaction().uniqueInteractionData().auxDateCreated().toString();
for (int i = 0; i < demographicFields.length; i++) {
Expand Down
8 changes: 2 additions & 6 deletions JeMPI_Apps/JeMPI_LibMPI/src/main/java/org/jembi/jempi/libmpi/dgraph/DgraphMutations.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
import org.jembi.jempi.shared.models.*;
import org.jembi.jempi.shared.utils.AppUtils;

import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
Expand All @@ -39,10 +38,7 @@ LinkInfo addNewDGraphInteraction(final Interaction interaction) {
result.interactionUID,
result.sourceUID,
1.0F,
new CustomUniqueGoldenRecordData(LocalDateTime.now(),
true,
interaction.uniqueInteractionData()
.auxId()));
new CustomUniqueGoldenRecordData(interaction.uniqueInteractionData()));
if (grUID == null) {
LOGGER.error("Failed to insert golden record");
return null;
Expand Down Expand Up @@ -309,7 +305,7 @@ Either<MpiGeneralError, LinkInfo> linkToNewGoldenRecord(
final var newGoldenID = cloneGoldenRecordFromInteraction(
interaction.demographicData(), interaction.interactionId(),
interaction.sourceId().uid(),
score, new CustomUniqueGoldenRecordData(LocalDateTime.now(), true, interaction.uniqueInteractionData().auxId()));
score, new CustomUniqueGoldenRecordData(interaction.uniqueInteractionData()));
return Either.right(new LinkInfo(newGoldenID, interactionId, score));
}

Expand Down
8 changes: 2 additions & 6 deletions JeMPI_Apps/JeMPI_LibMPI/src/main/java/org/jembi/jempi/libmpi/postgresql/LibPostgresql.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,11 +71,7 @@ private ExpandedInteraction findExpandedInteraction(final String eid) {
x.data()
.patient()))
.toList(),
new CustomUniqueGoldenRecordData(
LocalDateTime.now(),
true,
interaction.uniqueInteractionData()
.auxId()),
new CustomUniqueGoldenRecordData(interaction.uniqueInteractionData()),
goldenRecord.data()),
PostgresqlQueries.getScore(goldenRecord.uid(),
UUID.fromString(eid)))));
Expand All @@ -92,7 +88,7 @@ public GoldenRecord findGoldenRecord(final String goldenId) {
sourceIds.stream()
.map(x -> new CustomSourceId(x.id().toString(), x.data().facility(), x.data().patient()))
.toList(),
new CustomUniqueGoldenRecordData(LocalDateTime.now(), true, "AUX_ID"),
new CustomUniqueGoldenRecordData(null),
goldenRecord.data());
}

Expand Down
5 changes: 1 addition & 4 deletions ..._Apps/JeMPI_LibMPI/src/main/java/org/jembi/jempi/libmpi/postgresql/PostgresqlQueries.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Types;
import java.time.LocalDateTime;
import java.util.*;

import static org.jembi.jempi.libmpi.postgresql.PostgresqlMutations.*;
Expand Down Expand Up @@ -211,9 +210,7 @@ private static List<GoldenRecord> findCandidatesWorker(final String sql) {
CustomDemographicData.class));
list.add(new GoldenRecord(id,
null,
new CustomUniqueGoldenRecordData(LocalDateTime.now(),
true,
"AUX_ID"),
new CustomUniqueGoldenRecordData(null),
goldenRecordData));
}
return list;
Expand Down
10 changes: 10 additions & 0 deletions ...I_LibShared/src/main/java/org/jembi/jempi/shared/models/CustomUniqueGoldenRecordData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,18 @@

import com.fasterxml.jackson.annotation.JsonInclude;

import java.time.LocalDateTime;

@JsonInclude(JsonInclude.Include.NON_NULL)
public record CustomUniqueGoldenRecordData(java.time.LocalDateTime auxDateCreated,
Boolean auxAutoUpdateEnabled,
String auxId) {

public CustomUniqueGoldenRecordData(final CustomUniqueInteractionData uniqueInteractionData) {
this(LocalDateTime.now(),
true,
uniqueInteractionData.auxId()
);
}

}
5 changes: 1 addition & 4 deletions JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/models/GoldenRecord.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import com.fasterxml.jackson.annotation.JsonInclude;

import java.time.LocalDateTime;
import java.util.List;

@JsonInclude(JsonInclude.Include.NON_NULL)
Expand All @@ -15,9 +14,7 @@ public record GoldenRecord(
public GoldenRecord(final Interaction interaction) {
this(null,
List.of(interaction.sourceId()),
new CustomUniqueGoldenRecordData(LocalDateTime.now(),
true,
interaction.uniqueInteractionData().auxId()),
new CustomUniqueGoldenRecordData(interaction.uniqueInteractionData()),
interaction.demographicData());
}

Expand Down
10 changes: 0 additions & 10 deletions JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/linker/backend/LinkerDWH.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,16 +82,6 @@ static void helperUpdateInteractionsScore(
final LibMPI libMPI,
final float threshold,
final ExpandedGoldenRecord expandedGoldenRecord) {
if (LOGGER.isTraceEnabled()) {
expandedGoldenRecord.interactionsWithScore().forEach(interactionWithScore -> LOGGER.trace("{} -> {} : {}",
interactionWithScore.interaction()
.uniqueInteractionData()
.auxId(),
expandedGoldenRecord.goldenRecord()
.customUniqueGoldenRecordData()
.auxId(),
interactionWithScore.score()));
}
expandedGoldenRecord.interactionsWithScore().forEach(interactionWithScore -> {
final var interaction = interactionWithScore.interaction();
final var score =
Expand Down
7 changes: 5 additions & 2 deletions JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/stats/StatsTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@

import static java.lang.Math.min;
import static org.jembi.jempi.shared.utils.AppUtils.OBJECT_MAPPER;
import static org.jembi.jempi.shared.utils.AppUtils.isNullOrEmpty;

public final class StatsTask {

Expand Down Expand Up @@ -107,6 +106,7 @@ private List<ApiModels.ApiExpandedGoldenRecord> getGoldenRecordInteractions(fina
}

private void updateStatsDataSet(final ApiModels.ApiExpandedGoldenRecord expandedGoldenRecord) {
/*
final String goldenRecordAuxId = expandedGoldenRecord.goldenRecord().uniqueGoldenRecordData().auxId();
final String goldenRecordNumber = goldenRecordAuxId.substring(0, AUX_ID_SIGNIFICANT_CHARACTERS);

Expand All @@ -123,6 +123,7 @@ private void updateStatsDataSet(final ApiModels.ApiExpandedGoldenRecord expanded
} else {
entry.add(new GoldenRecordMembers(goldenRecordAuxId, list));
}
*/
}

private void processSubList(
Expand Down Expand Up @@ -156,6 +157,8 @@ public StatsResults run() {
LOGGER.info("Sub Lists: {}", subLists);
LOGGER.info("Final Sub List Size: {}", finalSubListSize);
}

/*
int fromIdx;
int toIdx;
for (long i = 0; i < subLists; i++) {
Expand Down Expand Up @@ -208,7 +211,7 @@ public StatsResults run() {
precision,
recall,
fScore);

*/
} catch (IOException e) {
LOGGER.error(e.getLocalizedMessage(), e);
}
Expand Down