Skip to content

Commit

Permalink
HDDS-11660. Recon List Key API: Reduce object creation and buffering …
Browse files Browse the repository at this point in the history 
…memory (apache#7405)
  • Loading branch information
@sodonnel
sodonnel authored Nov 8, 2024
1 parent 5d18b9c commit a7e3014
Show file tree
Hide file tree
Showing 7 changed files with 244 additions and 74 deletions.
33 changes: 27 additions & 6 deletions hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -291,11 +291,34 @@ public void untarCheckpointFile(File tarFile, Path destPath)
*/
public static String constructFullPath(OmKeyInfo omKeyInfo,
ReconNamespaceSummaryManager reconNamespaceSummaryManager,
ReconOMMetadataManager omMetadataManager)
throws IOException {
ReconOMMetadataManager omMetadataManager) throws IOException {
return constructFullPath(omKeyInfo.getKeyName(), omKeyInfo.getParentObjectID(), omKeyInfo.getVolumeName(),
omKeyInfo.getBucketName(), reconNamespaceSummaryManager, omMetadataManager);
}

StringBuilder fullPath = new StringBuilder(omKeyInfo.getKeyName());
long parentId = omKeyInfo.getParentObjectID();
/**
* Constructs the full path of a key from its key name and parent ID using a bottom-up approach, starting from the
* leaf node.
*
* The method begins with the leaf node (the key itself) and recursively prepends parent directory names, fetched
* via NSSummary objects, until reaching the parent bucket (parentId is -1). It effectively builds the path from
* bottom to top, finally prepending the volume and bucket names to complete the full path. If the directory structure
* is currently being rebuilt (indicated by the rebuildTriggered flag), this method returns an empty string to signify
* that path construction is temporarily unavailable.
*
* @param keyName The name of the key
* @param initialParentId The parent ID of the key
* @param volumeName The name of the volume
* @param bucketName The name of the bucket
* @return The constructed full path of the key as a String, or an empty string if a rebuild is in progress and
* the path cannot be constructed at this time.
* @throws IOException
*/
public static String constructFullPath(String keyName, long initialParentId, String volumeName, String bucketName,
ReconNamespaceSummaryManager reconNamespaceSummaryManager,
ReconOMMetadataManager omMetadataManager) throws IOException {
StringBuilder fullPath = new StringBuilder(keyName);
long parentId = initialParentId;
boolean isDirectoryPresent = false;

while (parentId != 0) {
Expand All @@ -320,8 +343,6 @@ public static String constructFullPath(OmKeyInfo omKeyInfo,
}

// Prepend the volume and bucket to the constructed path
String volumeName = omKeyInfo.getVolumeName();
String bucketName = omKeyInfo.getBucketName();
fullPath.insert(0, volumeName + OM_KEY_PREFIX + bucketName + OM_KEY_PREFIX);
if (isDirectoryPresent) {
return OmUtils.normalizeKey(fullPath.toString(), true);
Expand Down
95 changes: 39 additions & 56 deletions hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import org.apache.hadoop.ozone.recon.ReconUtils;
import org.apache.hadoop.ozone.recon.api.handlers.BucketHandler;
import org.apache.hadoop.ozone.recon.api.types.KeyEntityInfo;
import org.apache.hadoop.ozone.recon.api.types.KeyEntityInfoProtoWrapper;
import org.apache.hadoop.ozone.recon.api.types.KeyInsightInfoResponse;
import org.apache.hadoop.ozone.recon.api.types.ListKeysResponse;
import org.apache.hadoop.ozone.recon.api.types.NSSummary;
Expand All @@ -58,7 +59,6 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

Expand Down Expand Up @@ -989,7 +989,7 @@ public Response listKeys(@QueryParam("replicationType") String replicationType,
listKeysResponse = (ListKeysResponse) response.getEntity();
}

List<KeyEntityInfo> keyInfoList = listKeysResponse.getKeys();
List<KeyEntityInfoProtoWrapper> keyInfoList = listKeysResponse.getKeys();
if (!keyInfoList.isEmpty()) {
listKeysResponse.setLastKey(keyInfoList.get(keyInfoList.size() - 1).getKey());
}
Expand All @@ -1003,66 +1003,49 @@ private Response getListKeysResponse(ParamInfo paramInfo) {
listKeysResponse.setPath(paramInfo.getStartPrefix());
long replicatedTotal = 0;
long unreplicatedTotal = 0;
boolean keysFound = false; // Flag to track if any keys are found

// Search keys from non-FSO layout.
Map<String, OmKeyInfo> obsKeys;
Table<String, OmKeyInfo> keyTable =
omMetadataManager.getKeyTable(BucketLayout.LEGACY);
obsKeys = retrieveKeysFromTable(keyTable, paramInfo);
for (Map.Entry<String, OmKeyInfo> entry : obsKeys.entrySet()) {
keysFound = true;
KeyEntityInfo keyEntityInfo =
createKeyEntityInfoFromOmKeyInfo(entry.getKey(), entry.getValue());

listKeysResponse.getKeys().add(keyEntityInfo);
replicatedTotal += entry.getValue().getReplicatedSize();
unreplicatedTotal += entry.getValue().getDataSize();
}
Table<String, KeyEntityInfoProtoWrapper> keyTable =
omMetadataManager.getKeyTableLite(BucketLayout.LEGACY);
retrieveKeysFromTable(keyTable, paramInfo, listKeysResponse.getKeys());


// Search keys from FSO layout.
Map<String, OmKeyInfo> fsoKeys = searchKeysInFSO(paramInfo);
for (Map.Entry<String, OmKeyInfo> entry : fsoKeys.entrySet()) {
keysFound = true;
KeyEntityInfo keyEntityInfo =
createKeyEntityInfoFromOmKeyInfo(entry.getKey(), entry.getValue());

listKeysResponse.getKeys().add(keyEntityInfo);
replicatedTotal += entry.getValue().getReplicatedSize();
unreplicatedTotal += entry.getValue().getDataSize();
}
searchKeysInFSO(paramInfo, listKeysResponse.getKeys());

// If no keys were found, return a response indicating that no keys matched
if (!keysFound) {
if (listKeysResponse.getKeys().isEmpty()) {
return ReconResponseUtils.noMatchedKeysResponse(paramInfo.getStartPrefix());
}

for (KeyEntityInfoProtoWrapper keyEntityInfo : listKeysResponse.getKeys()) {
replicatedTotal += keyEntityInfo.getReplicatedSize();
unreplicatedTotal += keyEntityInfo.getSize();
}

// Set the aggregated totals in the response
listKeysResponse.setReplicatedDataSize(replicatedTotal);
listKeysResponse.setUnReplicatedDataSize(unreplicatedTotal);

return Response.ok(listKeysResponse).build();
} catch (IOException e) {
return ReconResponseUtils.createInternalServerErrorResponse(
"Error listing keys from OM DB: " + e.getMessage());
} catch (RuntimeException e) {
LOG.error("Error generating listKeys response", e);
return ReconResponseUtils.createInternalServerErrorResponse(
"Unexpected runtime error while searching keys in OM DB: " + e.getMessage());
} catch (Exception e) {
LOG.error("Error generating listKeys response", e);
return ReconResponseUtils.createInternalServerErrorResponse(
"Error listing keys from OM DB: " + e.getMessage());
}
}

public Map<String, OmKeyInfo> searchKeysInFSO(ParamInfo paramInfo)
public void searchKeysInFSO(ParamInfo paramInfo, List<KeyEntityInfoProtoWrapper> results)
throws IOException {
int originalLimit = paramInfo.getLimit();
Map<String, OmKeyInfo> matchedKeys = new LinkedHashMap<>();
// Convert the search prefix to an object path for FSO buckets
String startPrefixObjectPath = convertStartPrefixPathToObjectIdPath(paramInfo.getStartPrefix());
String[] names = parseRequestPath(startPrefixObjectPath);
Table<String, OmKeyInfo> fileTable =
omMetadataManager.getKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED);
Table<String, KeyEntityInfoProtoWrapper> fileTable =
omMetadataManager.getKeyTableLite(BucketLayout.FILE_SYSTEM_OPTIMIZED);

// If names.length > 2, then the search prefix is at the level above bucket level hence
// no need to find parent or extract id's or find subpaths as the fileTable is
Expand All @@ -1075,7 +1058,7 @@ public Map<String, OmKeyInfo> searchKeysInFSO(ParamInfo paramInfo)
NSSummary parentSummary =
reconNamespaceSummaryManager.getNSSummary(parentId);
if (parentSummary == null) {
return matchedKeys;
return;
}
List<String> subPaths = new ArrayList<>();
// Add the initial search prefix object path because it can have both files and subdirectories with files.
Expand All @@ -1087,21 +1070,17 @@ public Map<String, OmKeyInfo> searchKeysInFSO(ParamInfo paramInfo)
// Iterate over the subpaths and retrieve the files
for (String subPath : subPaths) {
paramInfo.setStartPrefix(subPath);
matchedKeys.putAll(
retrieveKeysFromTable(fileTable, paramInfo));
paramInfo.setLimit(originalLimit - matchedKeys.size());
if (matchedKeys.size() >= originalLimit) {
retrieveKeysFromTable(fileTable, paramInfo, results);
if (results.size() >= paramInfo.getLimit()) {
break;
}
}
return matchedKeys;
return;
}

paramInfo.setStartPrefix(startPrefixObjectPath);
// Iterate over for bucket and volume level search
matchedKeys.putAll(
retrieveKeysFromTable(fileTable, paramInfo));
return matchedKeys;
retrieveKeysFromTable(fileTable, paramInfo, results);
}


Expand Down Expand Up @@ -1174,32 +1153,31 @@ public String convertStartPrefixPathToObjectIdPath(String startPrefixPath)
* @return A map of keys and their corresponding OmKeyInfo objects.
* @throws IOException If there are problems accessing the table.
*/
private Map<String, OmKeyInfo> retrieveKeysFromTable(
Table<String, OmKeyInfo> table, ParamInfo paramInfo)
private void retrieveKeysFromTable(
Table<String, KeyEntityInfoProtoWrapper> table, ParamInfo paramInfo, List<KeyEntityInfoProtoWrapper> results)
throws IOException {
boolean skipPrevKey = false;
String seekKey = paramInfo.getPrevKey();
Map<String, OmKeyInfo> matchedKeys = new LinkedHashMap<>();
try (
TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>> keyIter = table.iterator()) {
TableIterator<String, ? extends Table.KeyValue<String, KeyEntityInfoProtoWrapper>> keyIter = table.iterator()) {

if (!paramInfo.isSkipPrevKeyDone() && isNotBlank(seekKey)) {
skipPrevKey = true;
Table.KeyValue<String, OmKeyInfo> seekKeyValue =
Table.KeyValue<String, KeyEntityInfoProtoWrapper> seekKeyValue =
keyIter.seek(seekKey);

// check if RocksDB was able to seek correctly to the given key prefix
// if not, then return empty result
// In case of an empty prevKeyPrefix, all the keys are returned
if (seekKeyValue == null || (!seekKeyValue.getKey().equals(paramInfo.getPrevKey()))) {
return matchedKeys;
return;
}
} else {
keyIter.seek(paramInfo.getStartPrefix());
}

while (keyIter.hasNext()) {
Table.KeyValue<String, OmKeyInfo> entry = keyIter.next();
Table.KeyValue<String, KeyEntityInfoProtoWrapper> entry = keyIter.next();
String dbKey = entry.getKey();
if (!dbKey.startsWith(paramInfo.getStartPrefix())) {
break; // Exit the loop if the key no longer matches the prefix
Expand All @@ -1209,9 +1187,14 @@ private Map<String, OmKeyInfo> retrieveKeysFromTable(
continue;
}
if (applyFilters(entry, paramInfo)) {
matchedKeys.put(dbKey, entry.getValue());
KeyEntityInfoProtoWrapper keyEntityInfo = entry.getValue();
keyEntityInfo.setKey(dbKey);
keyEntityInfo.setPath(ReconUtils.constructFullPath(keyEntityInfo.getKeyName(), keyEntityInfo.getParentId(),
keyEntityInfo.getVolumeName(), keyEntityInfo.getBucketName(), reconNamespaceSummaryManager,
omMetadataManager));
results.add(keyEntityInfo);
paramInfo.setLastKey(dbKey);
if (matchedKeys.size() >= paramInfo.getLimit()) {
if (results.size() >= paramInfo.getLimit()) {
break;
}
}
Expand All @@ -1220,10 +1203,10 @@ private Map<String, OmKeyInfo> retrieveKeysFromTable(
LOG.error("Error retrieving keys from table for path: {}", paramInfo.getStartPrefix(), exception);
throw exception;
}
return matchedKeys;
}

private boolean applyFilters(Table.KeyValue<String, OmKeyInfo> entry, ParamInfo paramInfo) throws IOException {
private boolean applyFilters(Table.KeyValue<String, KeyEntityInfoProtoWrapper> entry, ParamInfo paramInfo)
throws IOException {

LOG.debug("Applying filters on : {}", entry.getKey());

Expand All @@ -1238,7 +1221,7 @@ private boolean applyFilters(Table.KeyValue<String, OmKeyInfo> entry, ParamInfo
return false;
}

return entry.getValue().getDataSize() >= paramInfo.getKeySize();
return entry.getValue().getSize() >= paramInfo.getKeySize();
}

/**
Expand Down
Loading

0 comments on commit a7e3014

Please sign in to comment.