Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add more descriptive names to unmapped datatypes in data dictionary #23

Closed
wants to merge 4 commits into from
Closed

Add more descriptive names to unmapped datatypes in data dictionary #23

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
cd2dd15
Add more descriptive types to the data dictionary
foster33 Apr 23, 2024
883b6b0
Fix typo, change comment wording, remove '*'
foster33 Apr 24, 2024
bd3ade2
Remove now unused documentation
foster33 Apr 25, 2024
a37b8f1
Remove normalizationMap usage & fix test cases & introduce new setType()
foster33 May 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 12 additions & 6 deletions service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,11 @@
package datawave.microservice.metadata;

import java.io.IOException;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.time.LocalDate;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
Expand All @@ -23,6 +20,7 @@
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.user.WholeRowIterator;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.io.Text;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -266,13 +264,21 @@ private void setDescriptions() throws MarkingFunctions.Exception {
currField.getDescriptions().add(description);
}

// Set the normalized type for the current {@link DefaultMetadataField}. If no normalized version can be found for the type, the type will default to
// "Unknown".
// Receives a type that is not in the normalizationMap, which then is processed and returned.
// Ensures first letter of the type is always capitalized.
// Ensures redundant terminology like 'Type' is removed.
private String determineUnknownType(String unknown) {
String[] unknownType = unknown.split("\\.");
return StringUtils.capitalize(unknownType[unknownType.length - 1].replace("Type", ""));
}

// Set the normalized type for the current {@link DefaultMetadataField}. If the type cannot be found in the normalizationMap, the type will default to
// using a processed version of it's class name.
private void setType() {
int nullPos = currColumnQualifier.indexOf('\0');
String type = currColumnQualifier.substring(nullPos + 1);
String normalizedType = normalizationMap.get(type);
currField.addType(normalizedType != null ? normalizedType : "Unknown");
currField.addType(normalizedType != null ? normalizedType : determineUnknownType(type));
}

// Set the last updated date for the current {@link DefaultMetadataField} based on the timestamp of the current entry.
Expand Down
19 changes: 16 additions & 3 deletions service/src/test/java/datawave/microservice/metadata/DefaultMetadataFieldScannerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
Expand Down Expand Up @@ -109,12 +110,17 @@ public void whenRetrievingFields_givenNoDataTypeFilters_shouldReturnUnfilteredRe
contributorId.setDescription(Collections.singleton(createDescription("ContributorId Description")));
contributorId.setLastUpdated(DATE);

ArrayList<String> expectedTypes = new ArrayList<>();
expectedTypes.add("TestLcNoCookies*");
expectedTypes.add("TestNumberList*");
expectedTypes.add("Testleopard*");

DefaultMetadataField name = new DefaultMetadataField();
name.setFieldName("NAME");
name.setDataType("tvmaze");
name.setForwardIndexed(true);
name.setReverseIndexed(true);
name.setTypes(Collections.singletonList("Unknown"));
name.setTypes(expectedTypes);
name.setLastUpdated(DATE);

Collection<DefaultMetadataField> fields = scanner.getFields(Collections.emptyMap(), Collections.emptySet());
Expand Down Expand Up @@ -170,12 +176,17 @@ public void whenRetrievingFields_givenAliases_shouldReturnResultsWithAliases() t
contributorId.setDescription(Collections.singleton(createDescription("ContributorId Description")));
contributorId.setLastUpdated(DATE);

ArrayList<String> expectedTypes = new ArrayList<>();
expectedTypes.add("TestLcNoCookies*");
expectedTypes.add("TestNumberList*");
expectedTypes.add("Testleopard*");

DefaultMetadataField name = new DefaultMetadataField();
name.setFieldName("NAME");
name.setDataType("tvmaze");
name.setForwardIndexed(true);
name.setReverseIndexed(true);
name.setTypes(Collections.singletonList("Unknown"));
name.setTypes(expectedTypes);
name.setLastUpdated(DATE);

Map<String,String> aliases = new HashMap<>();
Expand Down Expand Up @@ -205,7 +216,9 @@ private void populateMetadataTable() throws TableNotFoundException, MutationsRej
name.put(new Text(ColumnFamilyConstants.COLF_E), new Text("tvmaze"), TIMESTAMP, new Value());
name.put(new Text(ColumnFamilyConstants.COLF_I), new Text("tvmaze"), TIMESTAMP, new Value());
name.put(new Text(ColumnFamilyConstants.COLF_RI), new Text("tvmaze"), TIMESTAMP, new Value());
name.put(new Text(ColumnFamilyConstants.COLF_T), new Text("tvmaze\0not.a.known.type"), TIMESTAMP, new Value());
name.put(new Text(ColumnFamilyConstants.COLF_T), new Text("tvmaze\0datawave.data.type.testNumberListType"), TIMESTAMP, new Value());
name.put(new Text(ColumnFamilyConstants.COLF_T), new Text("tvmaze\0datawave.data.type.testLcNoCookiesType"), TIMESTAMP, new Value());
name.put(new Text(ColumnFamilyConstants.COLF_T), new Text("tvmaze\0datawave.data.type.testleopardType"), TIMESTAMP, new Value());

BatchWriterConfig bwConfig = new BatchWriterConfig().setMaxMemory(10L).setMaxLatency(1, TimeUnit.SECONDS).setMaxWriteThreads(1);
BatchWriter writer = connector.createBatchWriter(METADATA_TABLE, bwConfig);
Expand Down
Loading