diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java index 830cf2e12dd..407694bf859 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java @@ -80,6 +80,7 @@ import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.recon.api.ServiceNotReadyException; import org.apache.hadoop.ozone.recon.api.types.NSSummary; import org.apache.hadoop.ozone.recon.api.types.DUResponse; import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; @@ -356,16 +357,14 @@ public static StringBuilder constructFullPathPrefix(long initialParentId, String if (nsSummary == null) { log.warn("NSSummary tree is currently being rebuilt or the directory could be in the progress of " + "deletion, returning empty string for path construction."); - fullPath.setLength(0); - return fullPath; + throw new ServiceNotReadyException("Service is initializing. Please try again later."); } if (nsSummary.getParentId() == -1) { if (rebuildTriggered.compareAndSet(false, true)) { triggerRebuild(reconNamespaceSummaryManager, omMetadataManager); } log.warn("NSSummary tree is currently being rebuilt, returning empty string for path construction."); - fullPath.setLength(0); - return fullPath; + throw new ServiceNotReadyException("Service is initializing. Please try again later."); } // On the last pass, dir-name will be empty and parent will be zero, indicating the loop should end. if (!nsSummary.getDirName().isEmpty()) { diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java index abd3fae4fa3..d7cb691253d 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java @@ -976,7 +976,7 @@ public Response listKeys(@QueryParam("replicationType") String replicationType, ListKeysResponse listKeysResponse = new ListKeysResponse(); if (!ReconUtils.isInitializationComplete(omMetadataManager)) { listKeysResponse.setStatus(ResponseStatus.INITIALIZING); - return Response.ok(listKeysResponse).build(); + return Response.status(Response.Status.SERVICE_UNAVAILABLE).entity(listKeysResponse).build(); } ParamInfo paramInfo = new ParamInfo(replicationType, creationDate, keySize, startPrefix, prevKey, limit, false, ""); @@ -997,9 +997,9 @@ public Response listKeys(@QueryParam("replicationType") String replicationType, } private Response getListKeysResponse(ParamInfo paramInfo) { + ListKeysResponse listKeysResponse = new ListKeysResponse(); try { paramInfo.setLimit(Math.max(0, paramInfo.getLimit())); // Ensure limit is non-negative - ListKeysResponse listKeysResponse = new ListKeysResponse(); listKeysResponse.setPath(paramInfo.getStartPrefix()); long replicatedTotal = 0; long unreplicatedTotal = 0; @@ -1009,7 +1009,6 @@ private Response getListKeysResponse(ParamInfo paramInfo) { omMetadataManager.getKeyTableLite(BucketLayout.LEGACY); retrieveKeysFromTable(keyTable, paramInfo, listKeysResponse.getKeys()); - // Search keys from FSO layout. searchKeysInFSO(paramInfo, listKeysResponse.getKeys()); @@ -1029,6 +1028,10 @@ private Response getListKeysResponse(ParamInfo paramInfo) { return Response.ok(listKeysResponse).build(); } catch (RuntimeException e) { + if (e instanceof ServiceNotReadyException) { + listKeysResponse.setStatus(ResponseStatus.INITIALIZING); + return Response.status(Response.Status.SERVICE_UNAVAILABLE).entity(listKeysResponse).build(); + } LOG.error("Error generating listKeys response", e); return ReconResponseUtils.createInternalServerErrorResponse( "Unexpected runtime error while searching keys in OM DB: " + e.getMessage()); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ServiceNotReadyException.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ServiceNotReadyException.java new file mode 100644 index 00000000000..4190cc279ce --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ServiceNotReadyException.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.api; + +/** + * This exception being thrown when Rest API service is still initializing and not yet ready. + */ +public class ServiceNotReadyException extends RuntimeException { + public ServiceNotReadyException(String message) { + super(message); + } +} + diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java index 3c39e4192d2..da5484c9b89 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java @@ -62,10 +62,12 @@ import org.apache.hadoop.ozone.recon.scm.ReconPipelineManager; import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade; import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager; +import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; import org.apache.hadoop.ozone.recon.tasks.ContainerKeyMapperTask; +import org.apache.hadoop.ozone.recon.tasks.NSSummaryTaskWithFSO; import org.hadoop.ozone.recon.schema.ContainerSchemaDefinition.UnHealthyContainerStates; import org.hadoop.ozone.recon.schema.tables.pojos.UnhealthyContainers; import org.junit.jupiter.api.BeforeEach; @@ -121,6 +123,7 @@ public class TestContainerEndpoint { LoggerFactory.getLogger(TestContainerEndpoint.class); private OzoneStorageContainerManager ozoneStorageContainerManager; + private ReconNamespaceSummaryManager reconNamespaceSummaryManager; private ReconContainerManager reconContainerManager; private ContainerStateManager containerStateManager; private ReconPipelineManager reconPipelineManager; @@ -198,6 +201,8 @@ private void initializeInjector() throws Exception { containerEndpoint = reconTestInjector.getInstance(ContainerEndpoint.class); containerHealthSchemaManager = reconTestInjector.getInstance(ContainerHealthSchemaManager.class); + this.reconNamespaceSummaryManager = + reconTestInjector.getInstance(ReconNamespaceSummaryManager.class); pipeline = getRandomPipeline(); pipelineID = pipeline.getId(); @@ -472,6 +477,10 @@ public void testGetKeysForContainer() throws IOException { // Now to check if the ContainerEndpoint also reads the File table // Set up test data for FSO keys setUpFSOData(); + NSSummaryTaskWithFSO nSSummaryTaskWithFso = + new NSSummaryTaskWithFSO(reconNamespaceSummaryManager, + reconOMMetadataManager, new OzoneConfiguration()); + nSSummaryTaskWithFso.reprocessWithFSO(reconOMMetadataManager); // Reprocess the container key mapper to ensure the latest mapping is used reprocessContainerKeyMapper(); response = containerEndpoint.getKeysForContainer(20L, -1, ""); @@ -556,6 +565,10 @@ public void testGetKeysForContainerWithPrevKey() throws IOException { setUpFSOData(); // Reprocess the container key mapper to ensure the latest mapping is used reprocessContainerKeyMapper(); + NSSummaryTaskWithFSO nSSummaryTaskWithFso = + new NSSummaryTaskWithFSO(reconNamespaceSummaryManager, + reconOMMetadataManager, new OzoneConfiguration()); + nSSummaryTaskWithFso.reprocessWithFSO(reconOMMetadataManager); response = containerEndpoint.getKeysForContainer(20L, -1, "/0/1/2/file7"); // Ensure that the expected number of keys is returned diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java index 54da926601e..9cda6d6e451 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java @@ -89,6 +89,7 @@ import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.mockito.Mockito.anyLong; @@ -791,8 +792,9 @@ public void testConstructFullPath() throws IOException { .setParentObjectID(DIR_TWO_OBJECT_ID) .build(); // Call constructFullPath and verify the result - fullPath = ReconUtils.constructFullPath(keyInfo, - reconNamespaceSummaryManager, reconOMMetadataManager); + OmKeyInfo finalKeyInfo = keyInfo; + assertThrows(ServiceNotReadyException.class, () -> ReconUtils.constructFullPath(finalKeyInfo, + reconNamespaceSummaryManager, reconOMMetadataManager)); } @Test @@ -813,8 +815,8 @@ public void testConstructFullPathWithNegativeParentIdTriggersRebuild() throws IO .setParentObjectID(dirOneObjectId) .build(); - String result = ReconUtils.constructFullPath(keyInfo, mockSummaryManager, mockMetadataManager); - assertEquals("", result, "Expected an empty string return due to rebuild trigger"); + assertThrows(ServiceNotReadyException.class, () -> + ReconUtils.constructFullPath(keyInfo, mockSummaryManager, mockMetadataManager)); } @Test @@ -836,7 +838,8 @@ public void testLoggingWhenParentIdIsNegative() throws IOException { .setParentObjectID(1L) .build(); - ReconUtils.constructFullPath(keyInfo, mockManager, null); + assertThrows(ServiceNotReadyException.class, () -> + ReconUtils.constructFullPath(keyInfo, mockManager, null)); // Assert ArgumentCaptor logCaptor = ArgumentCaptor.forClass(String.class); diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOmDBInsightEndPoint.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOmDBInsightEndPoint.java index 398d494ea0d..ed4b82fa3e0 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOmDBInsightEndPoint.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOmDBInsightEndPoint.java @@ -41,6 +41,7 @@ import org.apache.hadoop.ozone.recon.api.types.KeyInsightInfoResponse; import org.apache.hadoop.ozone.recon.api.types.ListKeysResponse; import org.apache.hadoop.ozone.recon.api.types.NSSummary; +import org.apache.hadoop.ozone.recon.api.types.ResponseStatus; import org.apache.hadoop.ozone.recon.persistence.AbstractReconSqlDBTest; import org.apache.hadoop.ozone.recon.persistence.ContainerHealthSchemaManager; import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; @@ -217,6 +218,7 @@ public class TestOmDBInsightEndPoint extends AbstractReconSqlDBTest { private static final long KEY_TWENTY_TWO_OBJECT_ID = 37L; private static final long KEY_TWENTY_THREE_OBJECT_ID = 38L; private static final long KEY_TWENTY_FOUR_OBJECT_ID = 39L; + private static final long KEY_TWENTY_FIVE_OBJECT_ID = 42L; private static final long EMPTY_OBS_BUCKET_OBJECT_ID = 40L; private static final long EMPTY_FSO_BUCKET_OBJECT_ID = 41L; @@ -242,6 +244,7 @@ public class TestOmDBInsightEndPoint extends AbstractReconSqlDBTest { private static final long KEY_SEVENTEEN_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 private static final long KEY_EIGHTEEN_SIZE = OzoneConsts.KB + 1; // bin 1 private static final long KEY_NINETEEN_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 + private static final long KEY_TWENTY_SIZE = OzoneConsts.KB + 1; // bin 1 private static final String OBS_BUCKET_PATH = "/volume1/obs-bucket"; private static final String FSO_BUCKET_PATH = "/volume1/fso-bucket"; @@ -1940,6 +1943,18 @@ public void testListKeysForEmptyOBSBucket() { assertEquals("", listKeysResponse.getLastKey()); } + @Test + public void testListKeysWhenNSSummaryNotInitialized() throws Exception { + reconNamespaceSummaryManager.clearNSSummaryTable(); + // bucket level DU + Response bucketResponse = + omdbInsightEndpoint.listKeys("RATIS", "", 0, FSO_BUCKET_TWO_PATH, + "", 1000); + ListKeysResponse listKeysResponse = (ListKeysResponse) bucketResponse.getEntity(); + assertEquals(ResponseStatus.INITIALIZING, listKeysResponse.getStatus()); + assertEquals(Response.Status.SERVICE_UNAVAILABLE.getStatusCode(), bucketResponse.getStatus()); + } + @Test public void testListKeysForEmptyFSOBucket() { Response bucketResponse = omdbInsightEndpoint.listKeys("RATIS", "", 0, EMPTY_FSO_BUCKET_PATH,