From ae3718be0fa11127cb272ddf305f8a0753f4603d Mon Sep 17 00:00:00 2001 From: Wellington Ramos Chevreuil Date: Tue, 5 Apr 2022 16:55:34 +0100 Subject: [PATCH] =?UTF-8?q?HBASE-26927=20Add=20snapshot=20scanner=20UT=20w?= =?UTF-8?q?ith=20SFT=20and=20some=20cleanups=20to=20Tes=E2=80=A6=20(#4322)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Duo Zhang Signed-off-by: Josh Elser --- .../client/TestTableSnapshotScanner.java | 39 ++++++++--------- .../TestTableSnapshotScannerWithSFT.java | 42 +++++++++++++++++++ 2 files changed, 59 insertions(+), 22 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScannerWithSFT.java diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java index 32eac9af0e11..976776ee3bbf 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hbase.client; -import java.io.FileNotFoundException; import java.io.IOException; import java.util.Arrays; import java.util.List; @@ -49,6 +48,7 @@ import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; @@ -73,6 +73,7 @@ public class TestTableSnapshotScanner { private FileSystem fs; private Path rootDir; + private boolean clusterUp; @Rule public TestName name = new TestName(); @@ -88,29 +89,30 @@ public static void blockUntilSplitFinished(HBaseTestingUtil util, TableName tabl } } + @Before public void setupCluster() throws Exception { setupConf(UTIL.getConfiguration()); StartTestingClusterOption option = StartTestingClusterOption.builder() .numRegionServers(NUM_REGION_SERVERS).numDataNodes(NUM_REGION_SERVERS) .createRootDir(true).build(); UTIL.startMiniCluster(option); + clusterUp = true; rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); fs = rootDir.getFileSystem(UTIL.getConfiguration()); } + @After public void tearDownCluster() throws Exception { - UTIL.shutdownMiniCluster(); + if (clusterUp) { + UTIL.shutdownMiniCluster(); + } } - private static void setupConf(Configuration conf) { + protected void setupConf(Configuration conf) { // Enable snapshot conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); } - @After - public void tearDown() throws Exception { - } - public static void createTableAndSnapshot(HBaseTestingUtil util, TableName tableName, String snapshotName, int numRegions) throws Exception { @@ -148,7 +150,6 @@ public static void createTableAndSnapshot(HBaseTestingUtil util, TableName table @Test public void testNoDuplicateResultsWhenSplitting() throws Exception { - setupCluster(); TableName tableName = TableName.valueOf("testNoDuplicateResultsWhenSplitting"); String snapshotName = "testSnapshotBug"; try { @@ -194,14 +195,12 @@ public void testNoDuplicateResultsWhenSplitting() throws Exception { } finally { UTIL.getAdmin().deleteSnapshot(snapshotName); UTIL.deleteTable(tableName); - tearDownCluster(); } } @Test public void testScanLimit() throws Exception { - setupCluster(); final TableName tableName = TableName.valueOf(name.getMethodName()); final String snapshotName = tableName + "Snapshot"; TableSnapshotScanner scanner = null; @@ -226,7 +225,6 @@ public void testScanLimit() throws Exception { } UTIL.getAdmin().deleteSnapshot(snapshotName); UTIL.deleteTable(tableName); - tearDownCluster(); } } @@ -247,7 +245,6 @@ public void testWithOfflineHBaseMultiRegion() throws Exception { @Test public void testScannerWithRestoreScanner() throws Exception { - setupCluster(); TableName tableName = TableName.valueOf("testScanner"); String snapshotName = "testScannerWithRestoreScanner"; try { @@ -287,19 +284,18 @@ public void testScannerWithRestoreScanner() throws Exception { } finally { UTIL.getAdmin().deleteSnapshot(snapshotName); UTIL.deleteTable(tableName); - tearDownCluster(); } } private void testScanner(HBaseTestingUtil util, String snapshotName, int numRegions, boolean shutdownCluster) throws Exception { - setupCluster(); TableName tableName = TableName.valueOf("testScanner"); try { createTableAndSnapshot(util, tableName, snapshotName, numRegions); if (shutdownCluster) { util.shutdownMiniHBaseCluster(); + clusterUp = false; } Path restoreDir = util.getDataTestDirOnTestFS(snapshotName); @@ -311,10 +307,9 @@ private void testScanner(HBaseTestingUtil util, String snapshotName, int numRegi verifyScanner(scanner, bbb, yyy); scanner.close(); } finally { - if (!shutdownCluster) { + if (clusterUp) { util.getAdmin().deleteSnapshot(snapshotName); util.deleteTable(tableName); - tearDownCluster(); } } } @@ -356,9 +351,9 @@ private static void verifyRow(Result result) throws IOException { } } + @Test public void testMergeRegion() throws Exception { - setupCluster(); TableName tableName = TableName.valueOf("testMergeRegion"); String snapshotName = tableName.getNameAsString() + "_snapshot"; Configuration conf = UTIL.getConfiguration(); @@ -384,9 +379,12 @@ public void testMergeRegion() throws Exception { for (RegionInfo region : regions) { Path regionDir = new Path(tableDir, region.getEncodedName()); for (Path familyDir : FSUtils.getFamilyDirs(fs, regionDir)) { - if (fs.listStatus(familyDir).length != 1) { - return false; + for (FileStatus fs : fs.listStatus(familyDir)) { + if(!fs.getPath().getName().equals(".filelist")) { + return true; + } } + return false; } } return true; @@ -463,14 +461,11 @@ public void testMergeRegion() throws Exception { } catch (Exception e) { LOG.error("scan snapshot error", e); Assert.fail("Should not throw Exception: " + e.getMessage()); - } finally { - tearDownCluster(); } } @Test public void testDeleteTableWithMergedRegions() throws Exception { - setupCluster(); final TableName tableName = TableName.valueOf(this.name.getMethodName()); String snapshotName = tableName.getNameAsString() + "_snapshot"; Configuration conf = UTIL.getConfiguration(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScannerWithSFT.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScannerWithSFT.java new file mode 100644 index 000000000000..e7eadcfb59e4 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScannerWithSFT.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; +import org.apache.hadoop.hbase.testclassification.ClientTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.junit.ClassRule; +import org.junit.experimental.categories.Category; + +@Category({ LargeTests.class, ClientTests.class}) +public class TestTableSnapshotScannerWithSFT extends TestTableSnapshotScanner { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestTableSnapshotScannerWithSFT.class); + + @Override + protected void setupConf(Configuration conf) { + super.setupConf(conf); + // Enable snapshot + conf.set(StoreFileTrackerFactory.TRACKER_IMPL, StoreFileTrackerFactory.Trackers.FILE.name()); + } + +}