From beed73be39baf0fd3641eb5a0b928c3316ca688b Mon Sep 17 00:00:00 2001 From: Wellington Ramos Chevreuil Date: Tue, 5 Apr 2022 16:55:34 +0100 Subject: [PATCH] =?UTF-8?q?HBASE-26927=20Add=20snapshot=20scanner=20UT=20w?= =?UTF-8?q?ith=20SFT=20and=20some=20cleanups=20to=20Tes=E2=80=A6=20(#4322)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Duo Zhang Signed-off-by: Josh Elser (cherry picked from commit ae3718be0fa11127cb272ddf305f8a0753f4603d) --- .../client/TestTableSnapshotScanner.java | 38 ++++++++--------- .../TestTableSnapshotScannerWithSFT.java | 42 +++++++++++++++++++ 2 files changed, 59 insertions(+), 21 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScannerWithSFT.java diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java index c51df964897e..de678e7185ed 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java @@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; @@ -74,6 +75,7 @@ public class TestTableSnapshotScanner { private FileSystem fs; private Path rootDir; + private boolean clusterUp; @Rule public TestName name = new TestName(); @@ -89,29 +91,30 @@ public static void blockUntilSplitFinished(HBaseTestingUtility util, TableName t } } + @Before public void setupCluster() throws Exception { setupConf(UTIL.getConfiguration()); StartMiniClusterOption option = StartMiniClusterOption.builder() .numRegionServers(NUM_REGION_SERVERS).numDataNodes(NUM_REGION_SERVERS) .createRootDir(true).build(); UTIL.startMiniCluster(option); + clusterUp = true; rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); fs = rootDir.getFileSystem(UTIL.getConfiguration()); } + @After public void tearDownCluster() throws Exception { - UTIL.shutdownMiniCluster(); + if (clusterUp) { + UTIL.shutdownMiniCluster(); + } } - private static void setupConf(Configuration conf) { + protected void setupConf(Configuration conf) { // Enable snapshot conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); } - @After - public void tearDown() throws Exception { - } - public static void createTableAndSnapshot(HBaseTestingUtility util, TableName tableName, String snapshotName, int numRegions) throws Exception { @@ -149,7 +152,6 @@ public static void createTableAndSnapshot(HBaseTestingUtility util, TableName ta @Test public void testNoDuplicateResultsWhenSplitting() throws Exception { - setupCluster(); TableName tableName = TableName.valueOf("testNoDuplicateResultsWhenSplitting"); String snapshotName = "testSnapshotBug"; try { @@ -195,14 +197,12 @@ public void testNoDuplicateResultsWhenSplitting() throws Exception { } finally { UTIL.getAdmin().deleteSnapshot(snapshotName); UTIL.deleteTable(tableName); - tearDownCluster(); } } @Test public void testScanLimit() throws Exception { - setupCluster(); final TableName tableName = TableName.valueOf(name.getMethodName()); final String snapshotName = tableName + "Snapshot"; TableSnapshotScanner scanner = null; @@ -227,7 +227,6 @@ public void testScanLimit() throws Exception { } UTIL.getAdmin().deleteSnapshot(snapshotName); UTIL.deleteTable(tableName); - tearDownCluster(); } } @@ -248,7 +247,6 @@ public void testWithOfflineHBaseMultiRegion() throws Exception { @Test public void testScannerWithRestoreScanner() throws Exception { - setupCluster(); TableName tableName = TableName.valueOf("testScanner"); String snapshotName = "testScannerWithRestoreScanner"; try { @@ -288,19 +286,18 @@ public void testScannerWithRestoreScanner() throws Exception { } finally { UTIL.getAdmin().deleteSnapshot(snapshotName); UTIL.deleteTable(tableName); - tearDownCluster(); } } private void testScanner(HBaseTestingUtility util, String snapshotName, int numRegions, boolean shutdownCluster) throws Exception { - setupCluster(); TableName tableName = TableName.valueOf("testScanner"); try { createTableAndSnapshot(util, tableName, snapshotName, numRegions); if (shutdownCluster) { util.shutdownMiniHBaseCluster(); + clusterUp = false; } Path restoreDir = util.getDataTestDirOnTestFS(snapshotName); @@ -312,10 +309,9 @@ private void testScanner(HBaseTestingUtility util, String snapshotName, int numR verifyScanner(scanner, bbb, yyy); scanner.close(); } finally { - if (!shutdownCluster) { + if (clusterUp) { util.getAdmin().deleteSnapshot(snapshotName); util.deleteTable(tableName); - tearDownCluster(); } } } @@ -359,7 +355,6 @@ private static void verifyRow(Result result) throws IOException { @Test public void testMergeRegion() throws Exception { - setupCluster(); TableName tableName = TableName.valueOf("testMergeRegion"); String snapshotName = tableName.getNameAsString() + "_snapshot"; Configuration conf = UTIL.getConfiguration(); @@ -385,9 +380,12 @@ public void testMergeRegion() throws Exception { for (RegionInfo region : regions) { Path regionDir = new Path(tableDir, region.getEncodedName()); for (Path familyDir : FSUtils.getFamilyDirs(fs, regionDir)) { - if (fs.listStatus(familyDir).length != 1) { - return false; + for (FileStatus fs : fs.listStatus(familyDir)) { + if(!fs.getPath().getName().equals(".filelist")) { + return true; + } } + return false; } } return true; @@ -462,17 +460,15 @@ public void testMergeRegion() throws Exception { } } catch (Exception e) { LOG.error("scan snapshot error", e); + Assert.fail("Should not throw Exception: " + e.getMessage()); Assert.fail("Should not throw FileNotFoundException"); Assert.assertTrue(e.getCause() != null); Assert.assertTrue(e.getCause().getCause() instanceof FileNotFoundException); - } finally { - tearDownCluster(); } } @Test public void testDeleteTableWithMergedRegions() throws Exception { - setupCluster(); final TableName tableName = TableName.valueOf(this.name.getMethodName()); String snapshotName = tableName.getNameAsString() + "_snapshot"; Configuration conf = UTIL.getConfiguration(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScannerWithSFT.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScannerWithSFT.java new file mode 100644 index 000000000000..e7eadcfb59e4 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScannerWithSFT.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; +import org.apache.hadoop.hbase.testclassification.ClientTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.junit.ClassRule; +import org.junit.experimental.categories.Category; + +@Category({ LargeTests.class, ClientTests.class}) +public class TestTableSnapshotScannerWithSFT extends TestTableSnapshotScanner { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestTableSnapshotScannerWithSFT.class); + + @Override + protected void setupConf(Configuration conf) { + super.setupConf(conf); + // Enable snapshot + conf.set(StoreFileTrackerFactory.TRACKER_IMPL, StoreFileTrackerFactory.Trackers.FILE.name()); + } + +}