From 2b6d557b095e7b09ffcaa30fe7157db81f6e4207 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Tue, 5 Nov 2024 17:01:22 +0000 Subject: [PATCH 1/2] HDDS-11649. Recon ListKeys API: Simplify filter predicates --- .../ozone/recon/api/OMDBInsightEndpoint.java | 52 ++++--------------- .../ozone/recon/api/types/ParamInfo.java | 14 +++++ 2 files changed, 24 insertions(+), 42 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java index d28275e5475..53e3c7427db 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java @@ -61,10 +61,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.TimeZone; -import java.util.function.Predicate; -import java.util.stream.Collectors; -import java.util.stream.Stream; import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; @@ -1231,46 +1227,18 @@ private boolean applyFilters(Table.KeyValue entry, ParamInfo LOG.debug("Applying filters on : {}", entry.getKey()); - long epochMillis = - ReconUtils.convertToEpochMillis(paramInfo.getCreationDate(), "MM-dd-yyyy HH:mm:ss", TimeZone.getDefault()); - Predicate> keyAgeFilter = keyData -> { - try { - return keyData.getValue().getCreationTime() >= epochMillis; - } catch (IOException e) { - throw new RuntimeException(e); - } - }; - Predicate> keyReplicationFilter = - keyData -> { - try { - return keyData.getValue().getReplicationConfig().getReplicationType().name() - .equals(paramInfo.getReplicationType()); - } catch (IOException e) { - try { - throw new IOException(e); - } catch (IOException ex) { - throw new RuntimeException(ex); - } - } - }; - Predicate> keySizeFilter = keyData -> { - try { - return keyData.getValue().getDataSize() >= paramInfo.getKeySize(); - } catch (IOException e) { - throw new RuntimeException(e); - } - }; - - List> filteredKeyList = Stream.of(entry) - .filter(keyData -> !StringUtils.isEmpty(paramInfo.getCreationDate()) ? keyAgeFilter.test(keyData) : true) - .filter( - keyData -> !StringUtils.isEmpty(paramInfo.getReplicationType()) ? keyReplicationFilter.test(keyData) : true) - .filter(keySizeFilter) - .collect(Collectors.toList()); + if (!StringUtils.isEmpty(paramInfo.getCreationDate()) + && !(entry.getValue().getCreationTime() >= paramInfo.getCreationDateEpoch())) { + return false; + } - LOG.debug("After applying filter on : {}, filtered list size: {}", entry.getKey(), filteredKeyList.size()); + if (!StringUtils.isEmpty(paramInfo.getReplicationType()) + && !entry.getValue().getReplicationConfig().getReplicationType().name().equals( + paramInfo.getReplicationType())) { + return false; + } - return (filteredKeyList.size() > 0); + return entry.getValue().getDataSize() >= paramInfo.getKeySize(); } /** diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ParamInfo.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ParamInfo.java index 345b0429076..e4bcea47b4d 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ParamInfo.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ParamInfo.java @@ -17,6 +17,10 @@ */ package org.apache.hadoop.ozone.recon.api.types; +import org.apache.hadoop.ozone.recon.ReconUtils; + +import java.util.TimeZone; + /** * Wrapper object for statistics of records of a page in API response. */ @@ -37,6 +41,8 @@ public class ParamInfo { */ private String creationDate; + private long creationDateEpoch = -1; + /** * */ @@ -87,6 +93,14 @@ public String getCreationDate() { return creationDate; } + public long getCreationDateEpoch() { + if (creationDateEpoch == -1) { + creationDateEpoch = ReconUtils.convertToEpochMillis( + getCreationDate(), "MM-dd-yyyy HH:mm:ss", TimeZone.getDefault()); + } + return creationDateEpoch; + } + public String getReplicationType() { return replicationType; } From 88d4cbbefcc17b63c40cdc0d78cb4a593afffd01 Mon Sep 17 00:00:00 2001 From: Stephen O'Donnell Date: Wed, 6 Nov 2024 15:17:49 +0000 Subject: [PATCH 2/2] Update hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java Co-authored-by: Doroszlai, Attila <6454655+adoroszlai@users.noreply.github.com> --- .../org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java index 53e3c7427db..8611abe88cd 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java @@ -1228,7 +1228,7 @@ private boolean applyFilters(Table.KeyValue entry, ParamInfo LOG.debug("Applying filters on : {}", entry.getKey()); if (!StringUtils.isEmpty(paramInfo.getCreationDate()) - && !(entry.getValue().getCreationTime() >= paramInfo.getCreationDateEpoch())) { + && (entry.getValue().getCreationTime() < paramInfo.getCreationDateEpoch())) { return false; }