From 8177556d139b5f7dddbb37ec55406d1172c2ede9 Mon Sep 17 00:00:00 2001 From: Mihir Monani Date: Fri, 22 Mar 2024 14:47:44 -0700 Subject: [PATCH] Mocked hbck and catalog janitor report object and variable name updates --- .../apache/hadoop/hbase/master/HMaster.java | 16 ++ .../http/hbck/model/HbckOverlapRegions.java | 18 +- .../http/hbck/model/HbckRegionHoles.java | 18 +- .../http/hbck/model/HbckUnknownServers.java | 11 +- .../master/http/TestHbckMetricsResource.java | 194 ++++++++++++++++-- 5 files changed, 217 insertions(+), 40 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index a03d9d76490f..d0d6e1ea2fd3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -1384,6 +1384,22 @@ private void finishActiveMasterInitialization() throws IOException, InterruptedE status.markComplete("Progress after master initialized complete"); } + /** + * Used for testing only to set Mock objects. + * @param hbckChore hbckChore + */ + public void setHbckChoreForTesting(HbckChore hbckChore) { + this.hbckChore = hbckChore; + } + + /** + * Used for testing only to set Mock objects. + * @param catalogJanitorChore catalogJanitorChore + */ + public void setCatalogJanitorChoreForTesting(CatalogJanitor catalogJanitorChore) { + this.catalogJanitorChore = catalogJanitorChore; + } + private void createMissingCFsInMetaDuringUpgrade(TableDescriptor metaDescriptor) throws IOException { TableDescriptor newMetaDesc = TableDescriptorBuilder.newBuilder(metaDescriptor) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/hbck/model/HbckOverlapRegions.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/hbck/model/HbckOverlapRegions.java index 84571219b1cf..cfa0340767be 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/hbck/model/HbckOverlapRegions.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/hbck/model/HbckOverlapRegions.java @@ -21,19 +21,19 @@ @InterfaceAudience.Private public class HbckOverlapRegions { - private final HbckRegionDetails region1; - private final HbckRegionDetails region2; + private final HbckRegionDetails region1Info; + private final HbckRegionDetails region2Info; - public HbckOverlapRegions(HbckRegionDetails region1, HbckRegionDetails region2) { - this.region1 = region1; - this.region2 = region2; + public HbckOverlapRegions(HbckRegionDetails region1Info, HbckRegionDetails region2Info) { + this.region1Info = region1Info; + this.region2Info = region2Info; } - public HbckRegionDetails getRegion1() { - return region1; + public HbckRegionDetails getRegion1Info() { + return region1Info; } - public HbckRegionDetails getRegion2() { - return region2; + public HbckRegionDetails getRegion2Info() { + return region2Info; } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/hbck/model/HbckRegionHoles.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/hbck/model/HbckRegionHoles.java index e96a48e61049..7b778872f544 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/hbck/model/HbckRegionHoles.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/hbck/model/HbckRegionHoles.java @@ -21,19 +21,19 @@ @InterfaceAudience.Private public class HbckRegionHoles { - private final HbckRegionDetails region1; - private final HbckRegionDetails region2; + private final HbckRegionDetails region1Info; + private final HbckRegionDetails region2Info; - public HbckRegionHoles(HbckRegionDetails regionHole1, HbckRegionDetails regionHole2) { - this.region1 = regionHole1; - this.region2 = regionHole2; + public HbckRegionHoles(HbckRegionDetails region1Info, HbckRegionDetails region2Info) { + this.region1Info = region1Info; + this.region2Info = region2Info; } - public HbckRegionDetails getRegion1() { - return region1; + public HbckRegionDetails getRegion1Info() { + return region1Info; } - public HbckRegionDetails getRegion2() { - return region2; + public HbckRegionDetails getRegion2Info() { + return region2Info; } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/hbck/model/HbckUnknownServers.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/hbck/model/HbckUnknownServers.java index 06148de94b1e..c291c8da64e2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/hbck/model/HbckUnknownServers.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/hbck/model/HbckUnknownServers.java @@ -21,17 +21,16 @@ @InterfaceAudience.Private public class HbckUnknownServers { - private final HbckRegionDetails serverRegionId; + private final HbckRegionDetails regionInfo; private final HbckServerName serverName; - public HbckUnknownServers(HbckRegionDetails unknownServerRegionId, - HbckServerName unknownServerName) { - this.serverRegionId = unknownServerRegionId; + public HbckUnknownServers(HbckRegionDetails regionInfo, HbckServerName unknownServerName) { + this.regionInfo = regionInfo; this.serverName = unknownServerName; } - public HbckRegionDetails getServerRegionId() { - return serverRegionId; + public HbckRegionDetails getRegionInfo() { + return regionInfo; } public HbckServerName getServerName() { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/http/TestHbckMetricsResource.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/http/TestHbckMetricsResource.java index 0420b4a7b705..6ed5087ccbdb 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/http/TestHbckMetricsResource.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/http/TestHbckMetricsResource.java @@ -17,16 +17,27 @@ */ package org.apache.hadoop.hbase.master.http; +import static org.apache.hadoop.hbase.client.RegionInfoBuilder.FIRST_META_REGIONINFO; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.startsWith; import static org.junit.Assert.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.concurrent.CompletableFuture; import java.util.function.Supplier; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.ConnectionRule; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; @@ -39,17 +50,27 @@ import org.apache.hadoop.hbase.client.AsyncConnection; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Durability; +import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.hbck.HbckChore; +import org.apache.hadoop.hbase.master.hbck.HbckReport; import org.apache.hadoop.hbase.master.http.hbck.resource.HbckMetricsResource; +import org.apache.hadoop.hbase.master.janitor.CatalogJanitor; +import org.apache.hadoop.hbase.master.janitor.CatalogJanitorReport; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.ExternalResource; import org.junit.rules.RuleChain; +import org.mockito.Mockito; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.javax.ws.rs.NotAcceptableException; import org.apache.hbase.thirdparty.javax.ws.rs.client.Client; @@ -63,6 +84,54 @@ @Category({ MasterTests.class, LargeTests.class }) public class TestHbckMetricsResource { + private static final Logger LOG = LoggerFactory.getLogger(TestHbckMetricsResource.class); + + // Test data for Mock HBCK Report + private static final long reportStartTime = 123456789000L; + private static final long reportEndTime = 234567890000L; + private static final String regionId1 = "regionId1"; + private static final String regionId2 = "regionId2"; + private static final String localhost1 = "localhost1"; + private static final String localhost2 = "localhost2"; + private static final String port = "16010"; + private static final String hostStartCode = "123456789"; + private static final String path1 = "hdfs://path1"; + private static final String path2 = "hdfs://path2"; + private static final String metaRegionID = FIRST_META_REGIONINFO.getEncodedName(); + private static final String metaTableName = FIRST_META_REGIONINFO.getTable().getNameAsString(); + + // Various Keys in HBCK JSON Response. + private static final String quoteColon = "\":"; + private static final String quote = "\""; + private static final String regionId = quote + "region_id" + quoteColon; + private static final String regionHdfsPath = quote + "region_hdfs_path" + quoteColon; + private static final String rsName = quote + "rs_name" + quoteColon; + private static final String hostName = quote + "host_name" + quoteColon; + private static final String hostPort = quote + "host_port" + quoteColon; + private static final String startCode = quote + "start_code" + quoteColon; + private static final String serverNameInMeta = quote + "server_name_in_meta" + quoteColon; + private static final String listOfServers = quote + "list_of_servers" + quoteColon; + private static final String region1Info = quote + "region1_info" + quoteColon; + private static final String region2Info = quote + "region2_info" + quoteColon; + private static final String regionInfo = quote + "region_info" + quoteColon; + private static final String serverName = quote + "server_name" + quoteColon; + private static final String tableName = quote + "table_name" + quoteColon; + + private static final String dataStartsWith = "{\"data\":["; + private static final String dataEndsWith = "]}"; + private static final String hbckReportStartTime = quote + "hbck_report_start_time" + quoteColon; + private static final String hbckReportEndTime = quote + "hbck_report_end_time" + quoteColon; + private static final String hbckOrphanRegionOnFS = + quote + "hbck_orphan_regions_on_fs" + quoteColon; + private static final String hbckOrphanRegionOnRS = + quote + "hbck_orphan_regions_on_rs" + quoteColon; + private static final String hbckInconsistentRegion = + quote + "hbck_inconsistent_regions" + quoteColon; + private static final String hbckHoles = quote + "hbck_holes" + quoteColon; + private static final String hbckOverlaps = quote + "hbck_overlaps" + quoteColon; + private static final String hbckUnknownServers = quote + "hbck_unknown_servers" + quoteColon; + private static final String hbckEmptyRegionInfo = quote + "hbck_empty_region_info" + quoteColon; + @ClassRule public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestHbckMetricsResource.class); @@ -75,9 +144,6 @@ public class TestHbckMetricsResource { final Configuration conf = HBaseConfiguration.create(); conf.setInt(HConstants.MASTER_INFO_PORT, 0); conf.set("hbase.http.jersey.tracing.type", "ON_DEMAND"); - // If we don't set this value to higher like 5sec, hbck chore won't get a chance to run and - // the all tests in this class will fail. - conf.setInt("hbase.master.hbck.chore.interval", 1000); return conf; }).build(); @@ -110,6 +176,68 @@ protected void before() throws Throwable { .setDurability(Durability.SKIP_WAL).build(); admin.createTable(tableDescriptor).get(); + HMaster master = miniClusterRule.getTestingUtility().getMiniHBaseCluster().getMaster(); + + HbckChore hbckChore = mock(HbckChore.class); + HbckReport hbckReport = mock(HbckReport.class); + CatalogJanitor catalogJanitorChore = mock(CatalogJanitor.class); + CatalogJanitorReport catalogJanitorReport = mock(CatalogJanitorReport.class); + master.setHbckChoreForTesting(hbckChore); + master.setCatalogJanitorChoreForTesting(catalogJanitorChore); + + // Test data for Mock HBCK Report + ServerName server1 = + ServerName.valueOf(localhost1, Integer.parseInt(port), Integer.parseInt(hostStartCode)); + ServerName server2 = + ServerName.valueOf(localhost2, Integer.parseInt(port), Integer.parseInt(hostStartCode)); + Path hdfsPath1 = new Path(path1); + Path hdfsPath2 = new Path(path2); + + // Orphan on RS Test data + Map mapOfOrphanRegionsOnRS = new HashMap<>(); + mapOfOrphanRegionsOnRS.put(regionId1, server1); + mapOfOrphanRegionsOnRS.put(regionId2, server2); + + // Orphan Region on FS Test Data + Map mapOfOrphanRegionOnFS = new HashMap<>(); + mapOfOrphanRegionOnFS.put(regionId1, hdfsPath1); + mapOfOrphanRegionOnFS.put(regionId2, hdfsPath2); + + // Inconsistent Regions Test Data + Map>> mapOfInconsistentRegions = new HashMap<>(); + mapOfInconsistentRegions.put(regionId1, new Pair<>(server1, Arrays.asList(server1, server2))); + mapOfInconsistentRegions.put(regionId2, new Pair<>(server2, Arrays.asList(server1, server2))); + + // Region Overlap and Region Holes Test Data + List> listOfRegion = new ArrayList<>(); + listOfRegion.add(new Pair<>(FIRST_META_REGIONINFO, FIRST_META_REGIONINFO)); + listOfRegion.add(new Pair<>(FIRST_META_REGIONINFO, FIRST_META_REGIONINFO)); + + // Unknown RegionServer Test Data + List> listOfUnknownServers = new ArrayList<>(); + listOfUnknownServers.add(new Pair<>(FIRST_META_REGIONINFO, server1)); + listOfUnknownServers.add(new Pair<>(FIRST_META_REGIONINFO, server2)); + + // Empty Region Info Test Data + List listOfEmptyRegionInfo = new ArrayList<>(); + listOfEmptyRegionInfo.add(regionId1.getBytes()); + listOfEmptyRegionInfo.add(regionId2.getBytes()); + + // Mock HBCK Report and CatalogJanitor Report + when(hbckReport.getCheckingStartTimestamp()) + .thenReturn(Instant.ofEpochMilli(reportStartTime)); + when(hbckReport.getCheckingEndTimestamp()).thenReturn(Instant.ofEpochSecond(reportEndTime)); + when(hbckReport.getOrphanRegionsOnFS()).thenReturn(mapOfOrphanRegionOnFS); + when(hbckReport.getOrphanRegionsOnRS()).thenReturn(mapOfOrphanRegionsOnRS); + when(hbckReport.getInconsistentRegions()).thenReturn(mapOfInconsistentRegions); + when(catalogJanitorReport.getHoles()).thenReturn(listOfRegion); + when(catalogJanitorReport.getOverlaps()).thenReturn(listOfRegion); + when(catalogJanitorReport.getUnknownServers()).thenReturn(listOfUnknownServers); + when(catalogJanitorReport.getEmptyRegionInfo()).thenReturn(listOfEmptyRegionInfo); + + Mockito.doReturn(hbckReport).when(hbckChore).getLastReport(); + Mockito.doReturn(catalogJanitorReport).when(catalogJanitorChore).getLastReport(); + final String baseUrl = admin.getMaster().thenApply(ServerName::getHostname).thenCombine(admin.getMasterInfoPort(), (hostName, infoPort) -> "http://" + hostName + ":" + infoPort).get(); @@ -143,12 +271,14 @@ protected void after() { public void testGetRoot() { final String response = classRule.getTarget().request(MediaType.APPLICATION_JSON_TYPE) .header("X-Jersey-Tracing-Accept", true).get(String.class); - assertThat(response, allOf(containsString("\"hbck_report_start_time\":"), - containsString("\"hbck_report_end_time\":"), containsString("\"hbck_orphan_regions_on_fs\":"), - containsString("\"hbck_orphan_regions_on_rs\":"), - containsString("\"hbck_inconsistent_regions\":"), containsString("\"hbck_holes\":"), - containsString("\"hbck_overlaps\":"), containsString("\"hbck_unknown_servers\":"), - containsString("\"hbck_empty_region_info\":"))); + LOG.info("HBCK JSON Response : " + response); + assertThat(response, + allOf(containsString(hbckReportStartTime), containsString(hbckReportEndTime), + containsString(hbckOrphanRegionOnFS), containsString(hbckOrphanRegionOnRS), + containsString(hbckInconsistentRegion), containsString(hbckHoles), + containsString(hbckOverlaps), containsString(hbckUnknownServers), + containsString(hbckEmptyRegionInfo), containsString(Objects.toString(reportStartTime)), + containsString(Objects.toString(reportEndTime)))); } @Test @@ -162,7 +292,11 @@ public void testGetOrphanRegionOnFS() { final String response = classRule.getTarget().path("orphan-regions-on-fs").request(MediaType.APPLICATION_JSON_TYPE) .header("X-Jersey-Tracing-Accept", true).get(String.class); - assertThat(response, allOf(startsWith("{\"data\":["), endsWith("]}"))); + LOG.info("HBCK Response for resource orphan-regions-on-fs : " + response); + assertThat(response, + allOf(startsWith(dataStartsWith), endsWith(dataEndsWith), containsString(regionId), + containsString(regionHdfsPath), containsString(regionId1), containsString(regionId2), + containsString(path1), containsString(path2))); } @Test @@ -177,7 +311,13 @@ public void testGetOrphanRegionOnRS() { final String response = classRule.getTarget().path("orphan-regions-on-rs").request(MediaType.APPLICATION_JSON_TYPE) .header("X-Jersey-Tracing-Accept", true).get(String.class); - assertThat(response, allOf(startsWith("{\"data\":["), endsWith("]}"))); + LOG.info("HBCK Response for resource orphan-regions-on-rs : " + response); + assertThat(response, + allOf(startsWith(dataStartsWith), endsWith(dataEndsWith), containsString(regionId), + containsString(rsName), containsString(hostName), containsString(hostPort), + containsString(startCode), containsString(regionId1), containsString(regionId2), + containsString(localhost1), containsString(localhost2), containsString(port), + containsString(hostStartCode))); } @Test @@ -192,7 +332,13 @@ public void testGetInconsistentRegions() { final String response = classRule.getTarget().path("inconsistent-regions").request(MediaType.APPLICATION_JSON_TYPE) .header("X-Jersey-Tracing-Accept", true).get(String.class); - assertThat(response, allOf(startsWith("{\"data\":["), endsWith("]}"))); + LOG.info("HBCK Response for resource inconsistent-regions : " + response); + assertThat(response, + allOf(startsWith(dataStartsWith), endsWith(dataEndsWith), containsString(hostName), + containsString(hostPort), containsString(startCode), containsString(listOfServers), + containsString(regionId1), containsString(regionId2), containsString(regionId), + containsString(serverNameInMeta), containsString(localhost1), containsString(localhost2), + containsString(port), containsString(hostStartCode))); } @Test @@ -207,7 +353,11 @@ public void testGetRegionHoles() { final String response = classRule.getTarget().path("region-holes").request(MediaType.APPLICATION_JSON_TYPE) .header("X-Jersey-Tracing-Accept", true).get(String.class); - assertThat(response, allOf(startsWith("{\"data\":["), endsWith("]}"))); + LOG.info("HBCK Response for resource region-holes : " + response); + assertThat(response, + allOf(startsWith(dataStartsWith), endsWith(dataEndsWith), containsString(region1Info), + containsString(region2Info), containsString(regionId), containsString(tableName), + containsString(metaRegionID), containsString(metaTableName))); } @Test @@ -221,7 +371,11 @@ public void testGetRegionOverlaps() { final String response = classRule.getTarget().path("region-overlaps").request(MediaType.APPLICATION_JSON_TYPE) .header("X-Jersey-Tracing-Accept", true).get(String.class); - assertThat(response, allOf(startsWith("{\"data\":["), endsWith("]}"))); + LOG.info("HBCK Response for resource region-overlaps : " + response); + assertThat(response, + allOf(startsWith(dataStartsWith), endsWith(dataEndsWith), containsString(regionId), + containsString(tableName), containsString(region2Info), containsString(region2Info), + containsString(metaRegionID), containsString(metaTableName))); } @Test @@ -235,7 +389,13 @@ public void testGetUnkownServers() { final String response = classRule.getTarget().path("unknown-servers").request(MediaType.APPLICATION_JSON_TYPE) .header("X-Jersey-Tracing-Accept", true).get(String.class); - assertThat(response, allOf(startsWith("{\"data\":["), endsWith("]}"))); + LOG.info("HBCK Response for resource unknown-servers : " + response); + assertThat(response, + allOf(startsWith(dataStartsWith), endsWith(dataEndsWith), containsString(regionInfo), + containsString(regionId), containsString(tableName), containsString(serverName), + containsString(serverName), containsString(port), containsString(startCode), + containsString(metaRegionID), containsString(metaTableName), containsString(localhost1), + containsString(localhost2), containsString(port), containsString(startCode))); } @Test @@ -249,7 +409,9 @@ public void testGetEmptyRegionInfo() { final String response = classRule.getTarget().path("empty-regioninfo").request(MediaType.APPLICATION_JSON_TYPE) .header("X-Jersey-Tracing-Accept", true).get(String.class); - assertThat(response, allOf(startsWith("{\"data\":["), endsWith("]}"))); + LOG.info("HBCK Response for resource empty-regioninfo : " + response); + assertThat(response, allOf(startsWith(dataStartsWith), endsWith(dataEndsWith), + containsString(regionInfo), containsString(regionId1), containsString(regionId2))); } @Test