Skip to content

Commit

Permalink
HDDS-10634. Recon - listKeys API for listing keys with optional filte…
Browse files Browse the repository at this point in the history
…rs (#6658)
  • Loading branch information
devmadhuu authored May 26, 2024
1 parent aed1d81 commit 40951a4
Show file tree
Hide file tree
Showing 12 changed files with 2,304 additions and 34 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ private ReconConstants() {
public static final String DEFAULT_OPEN_KEY_INCLUDE_NON_FSO = "false";
public static final String DEFAULT_OPEN_KEY_INCLUDE_FSO = "false";
public static final String DEFAULT_FETCH_COUNT = "1000";
public static final String DEFAULT_KEY_SIZE = "0";
public static final String DEFAULT_BATCH_NUMBER = "1";
public static final String RECON_QUERY_BATCH_PARAM = "batchNum";
public static final String RECON_QUERY_PREVKEY = "prevKey";
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.hadoop.ozone.recon;

import com.google.inject.Singleton;

import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;

/**
* Recon API Response Utility class.
*/
@Singleton
public final class ReconResponseUtils {

// Declared a private constructor to avoid checkstyle issues.
private ReconResponseUtils() {

}

/**
* Returns a response indicating that no keys matched the search prefix.
*
* @param startPrefix The search prefix that was used.
* @return The response indicating that no keys matched the search prefix.
*/
public static Response noMatchedKeysResponse(String startPrefix) {
String jsonResponse = String.format(
"{\"message\": \"No keys matched the search prefix: '%s'.\"}",
startPrefix);
return Response.status(Response.Status.NOT_FOUND)
.entity(jsonResponse)
.type(MediaType.APPLICATION_JSON)
.build();
}

/**
* Utility method to create a bad request response with a custom message.
* Which means the request sent by the client to the server is incorrect
* or malformed and cannot be processed by the server.
*
* @param message The message to include in the response body.
* @return A Response object configured with the provided message.
*/
public static Response createBadRequestResponse(String message) {
String jsonResponse = String.format("{\"message\": \"%s\"}", message);
return Response.status(Response.Status.BAD_REQUEST)
.entity(jsonResponse)
.type(MediaType.APPLICATION_JSON)
.build();
}

/**
* Utility method to create an internal server error response with a custom message.
* Which means the server encountered an unexpected condition that prevented it
* from fulfilling the request.
*
* @param message The message to include in the response body.
* @return A Response object configured with the provided message.
*/
public static Response createInternalServerErrorResponse(String message) {
String jsonResponse = String.format("{\"message\": \"%s\"}", message);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(jsonResponse)
.type(MediaType.APPLICATION_JSON)
.build();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,13 @@
import java.nio.file.Path;
import java.nio.file.Paths;
import java.sql.Timestamp;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.TimeZone;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
Expand All @@ -39,6 +44,7 @@

import com.google.common.base.Preconditions;
import com.google.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
Expand All @@ -65,6 +71,8 @@
import static org.jooq.impl.DSL.using;

import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.recon.api.types.NSSummary;
import org.apache.hadoop.ozone.recon.api.types.DUResponse;
Expand All @@ -80,6 +88,8 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.ws.rs.core.Response;

/**
* Recon Utility class.
*/
Expand Down Expand Up @@ -509,4 +519,102 @@ public SCMNodeDetails getReconNodeDetails(OzoneConfiguration conf) {
public static void setLogger(Logger logger) {
log = logger;
}

/**
* Return if all OMDB tables that will be used are initialized.
* @return if tables are initialized
*/
public static boolean isInitializationComplete(ReconOMMetadataManager omMetadataManager) {
if (omMetadataManager == null) {
return false;
}
return omMetadataManager.getVolumeTable() != null
&& omMetadataManager.getBucketTable() != null
&& omMetadataManager.getDirectoryTable() != null
&& omMetadataManager.getFileTable() != null
&& omMetadataManager.getKeyTable(BucketLayout.LEGACY) != null;
}

/**
* Converts string date in a provided format to server timezone's epoch milllioseconds.
*
* @param dateString
* @param dateFormat
* @param timeZone
* @return the epoch milliseconds representation of the date.
* @throws ParseException
*/
public static long convertToEpochMillis(String dateString, String dateFormat, TimeZone timeZone) {
String localDateFormat = dateFormat;
try {
if (StringUtils.isEmpty(dateString)) {
return Instant.now().toEpochMilli();
}
if (StringUtils.isEmpty(dateFormat)) {
localDateFormat = "MM-dd-yyyy HH:mm:ss";
}
if (null == timeZone) {
timeZone = TimeZone.getDefault();
}
SimpleDateFormat sdf = new SimpleDateFormat(localDateFormat);
sdf.setTimeZone(timeZone); // Set server's timezone
Date date = sdf.parse(dateString);
return date.getTime(); // Convert to epoch milliseconds
} catch (ParseException parseException) {
log.error("Date parse exception for date: {} in format: {} -> {}", dateString, localDateFormat, parseException);
return Instant.now().toEpochMilli();
} catch (Exception exception) {
log.error("Unexpected error while parsing date: {} in format: {} -> {}", dateString, localDateFormat, exception);
return Instant.now().toEpochMilli();
}
}

/**
* Validates volume or bucket names according to specific rules.
*
* @param resName The name to validate (volume or bucket).
* @return A Response object if validation fails, or null if the name is valid.
*/
public static Response validateNames(String resName)
throws IllegalArgumentException {
if (resName.length() < OzoneConsts.OZONE_MIN_BUCKET_NAME_LENGTH ||
resName.length() > OzoneConsts.OZONE_MAX_BUCKET_NAME_LENGTH) {
throw new IllegalArgumentException(
"Bucket or Volume name length should be between " +
OzoneConsts.OZONE_MIN_BUCKET_NAME_LENGTH + " and " +
OzoneConsts.OZONE_MAX_BUCKET_NAME_LENGTH);
}

if (resName.charAt(0) == '.' || resName.charAt(0) == '-' ||
resName.charAt(resName.length() - 1) == '.' ||
resName.charAt(resName.length() - 1) == '-') {
throw new IllegalArgumentException(
"Bucket or Volume name cannot start or end with " +
"hyphen or period");
}

// Regex to check for lowercase letters, numbers, hyphens, underscores, and periods only.
if (!resName.matches("^[a-z0-9._-]+$")) {
throw new IllegalArgumentException(
"Bucket or Volume name can only contain lowercase " +
"letters, numbers, hyphens, underscores, and periods");
}

// If all checks pass, the name is valid
return null;
}

/**
* Constructs an object path with the given IDs.
*
* @param ids The IDs to construct the object path with.
* @return The constructed object path.
*/
public static String constructObjectPathWithPrefix(long... ids) {
StringBuilder pathBuilder = new StringBuilder();
for (long id : ids) {
pathBuilder.append(OM_KEY_PREFIX).append(id);
}
return pathBuilder.toString();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
package org.apache.hadoop.ozone.recon.api;

import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.recon.ReconUtils;
import org.apache.hadoop.ozone.recon.api.handlers.EntityHandler;
import org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse;
import org.apache.hadoop.ozone.recon.api.types.DUResponse;
Expand Down Expand Up @@ -78,7 +78,7 @@ public Response getBasicInfo(
}

NamespaceSummaryResponse namespaceSummaryResponse;
if (!isInitializationComplete()) {
if (!ReconUtils.isInitializationComplete(omMetadataManager)) {
namespaceSummaryResponse =
NamespaceSummaryResponse.newBuilder()
.setEntityType(EntityType.UNKNOWN)
Expand Down Expand Up @@ -119,7 +119,7 @@ public Response getDiskUsage(@QueryParam("path") String path,
}

DUResponse duResponse = new DUResponse();
if (!isInitializationComplete()) {
if (!ReconUtils.isInitializationComplete(omMetadataManager)) {
duResponse.setStatus(ResponseStatus.INITIALIZING);
return Response.ok(duResponse).build();
}
Expand Down Expand Up @@ -150,7 +150,7 @@ public Response getQuotaUsage(@QueryParam("path") String path)
}

QuotaUsageResponse quotaUsageResponse = new QuotaUsageResponse();
if (!isInitializationComplete()) {
if (!ReconUtils.isInitializationComplete(omMetadataManager)) {
quotaUsageResponse.setResponseCode(ResponseStatus.INITIALIZING);
return Response.ok(quotaUsageResponse).build();
}
Expand Down Expand Up @@ -181,7 +181,7 @@ public Response getFileSizeDistribution(@QueryParam("path") String path)

FileSizeDistributionResponse distResponse =
new FileSizeDistributionResponse();
if (!isInitializationComplete()) {
if (!ReconUtils.isInitializationComplete(omMetadataManager)) {
distResponse.setStatus(ResponseStatus.INITIALIZING);
return Response.ok(distResponse).build();
}
Expand All @@ -195,19 +195,4 @@ public Response getFileSizeDistribution(@QueryParam("path") String path)
return Response.ok(distResponse).build();
}

/**
* Return if all OMDB tables that will be used are initialized.
* @return if tables are initialized
*/
private boolean isInitializationComplete() {
if (omMetadataManager == null) {
return false;
}
return omMetadataManager.getVolumeTable() != null
&& omMetadataManager.getBucketTable() != null
&& omMetadataManager.getDirectoryTable() != null
&& omMetadataManager.getFileTable() != null
&& omMetadataManager.getKeyTable(BucketLayout.LEGACY) != null;
}

}
Loading

0 comments on commit 40951a4

Please sign in to comment.