diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java index 42d0118ba657..573ff6bb9d9f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; + import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; @@ -29,14 +30,13 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletionException; + import org.apache.hadoop.hbase.AsyncMetaTableAccessor; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.TableExistsException; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.master.LoadBalancer; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; @@ -52,6 +52,7 @@ * @see TestAsyncTableAdminApi2 This test and it used to be joined it was taking longer than our * ten minute timeout so they were split. * @see TestAsyncTableAdminApi3 Another split out from this class so each runs under ten minutes. + * @see TestAsyncTableAdminApi4 Another split out from this class so each runs under ten minutes. */ @RunWith(Parameterized.class) @Category({ LargeTests.class, ClientTests.class }) @@ -361,96 +362,4 @@ private void testTruncateTable(final TableName tableName, boolean preserveSplits assertEquals(1, TEST_UTIL.getHBaseCluster().getRegions(tableName).size()); } } - - @Test - public void testCloneTableSchema() throws Exception { - final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new"); - testCloneTableSchema(tableName, newTableName, false); - } - - @Test - public void testCloneTableSchemaPreservingSplits() throws Exception { - final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new"); - testCloneTableSchema(tableName, newTableName, true); - } - - private void testCloneTableSchema(final TableName tableName, - final TableName newTableName, boolean preserveSplits) throws Exception { - byte[][] splitKeys = new byte[2][]; - splitKeys[0] = Bytes.toBytes(4); - splitKeys[1] = Bytes.toBytes(8); - int NUM_FAMILYS = 2; - int NUM_REGIONS = 3; - int BLOCK_SIZE = 1024; - int TTL = 86400; - boolean BLOCK_CACHE = false; - - // Create the table - TableDescriptor tableDesc = TableDescriptorBuilder - .newBuilder(tableName) - .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY_0)) - .setColumnFamily(ColumnFamilyDescriptorBuilder - .newBuilder(FAMILY_1) - .setBlocksize(BLOCK_SIZE) - .setBlockCacheEnabled(BLOCK_CACHE) - .setTimeToLive(TTL) - .build()).build(); - admin.createTable(tableDesc, splitKeys).join(); - - assertEquals(NUM_REGIONS, TEST_UTIL.getHBaseCluster().getRegions(tableName).size()); - assertTrue("Table should be created with splitKyes + 1 rows in META", - admin.isTableAvailable(tableName, splitKeys).get()); - - // Clone & Verify - admin.cloneTableSchema(tableName, newTableName, preserveSplits).join(); - TableDescriptor newTableDesc = admin.getDescriptor(newTableName).get(); - - assertEquals(NUM_FAMILYS, newTableDesc.getColumnFamilyCount()); - assertEquals(BLOCK_SIZE, newTableDesc.getColumnFamily(FAMILY_1).getBlocksize()); - assertEquals(BLOCK_CACHE, newTableDesc.getColumnFamily(FAMILY_1).isBlockCacheEnabled()); - assertEquals(TTL, newTableDesc.getColumnFamily(FAMILY_1).getTimeToLive()); - TEST_UTIL.verifyTableDescriptorIgnoreTableName(tableDesc, newTableDesc); - - if (preserveSplits) { - assertEquals(NUM_REGIONS, TEST_UTIL.getHBaseCluster().getRegions(newTableName).size()); - assertTrue("New table should be created with splitKyes + 1 rows in META", - admin.isTableAvailable(newTableName, splitKeys).get()); - } else { - assertEquals(1, TEST_UTIL.getHBaseCluster().getRegions(newTableName).size()); - } - } - - @Test - public void testCloneTableSchemaWithNonExistentSourceTable() throws Exception { - final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new"); - // test for non-existent source table - try { - admin.cloneTableSchema(tableName, newTableName, false).join(); - fail("Should have failed when source table doesn't exist."); - } catch (CompletionException e) { - assertTrue(e.getCause() instanceof TableNotFoundException); - } - } - - @Test - public void testCloneTableSchemaWithExistentDestinationTable() throws Exception { - final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new"); - byte[] FAMILY_0 = Bytes.toBytes("cf0"); - TEST_UTIL.createTable(tableName, FAMILY_0); - TEST_UTIL.createTable(newTableName, FAMILY_0); - // test for existent destination table - try { - admin.cloneTableSchema(tableName, newTableName, false).join(); - fail("Should have failed when destination table exists."); - } catch (CompletionException e) { - assertTrue(e.getCause() instanceof TableExistsException); - } - } - - @Test - public void testIsTableAvailableWithInexistantTable() throws Exception { - final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new"); - // test for inexistant table - assertFalse(admin.isTableAvailable(newTableName).get()); - } -} +} \ No newline at end of file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi4.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi4.java new file mode 100644 index 000000000000..7daf049abc7f --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi4.java @@ -0,0 +1,143 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.util.concurrent.CompletionException; + +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.TableExistsException; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.testclassification.ClientTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +/** + * Class to test asynchronous table admin operations + * @see TestAsyncTableAdminApi Split from it so each runs under ten minutes. + */ +@RunWith(Parameterized.class) +@Category({ LargeTests.class, ClientTests.class }) +public class TestAsyncTableAdminApi4 extends TestAsyncAdminBase { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestAsyncTableAdminApi4.class); + + @Test + public void testCloneTableSchema() throws Exception { + final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new"); + testCloneTableSchema(tableName, newTableName, false); + } + + @Test + public void testCloneTableSchemaPreservingSplits() throws Exception { + final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new"); + testCloneTableSchema(tableName, newTableName, true); + } + + private void testCloneTableSchema(final TableName tableName, + final TableName newTableName, boolean preserveSplits) throws Exception { + byte[][] splitKeys = new byte[2][]; + splitKeys[0] = Bytes.toBytes(4); + splitKeys[1] = Bytes.toBytes(8); + int NUM_FAMILYS = 2; + int NUM_REGIONS = 3; + int BLOCK_SIZE = 1024; + int TTL = 86400; + boolean BLOCK_CACHE = false; + + // Create the table + TableDescriptor tableDesc = TableDescriptorBuilder + .newBuilder(tableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY_0)) + .setColumnFamily(ColumnFamilyDescriptorBuilder + .newBuilder(FAMILY_1) + .setBlocksize(BLOCK_SIZE) + .setBlockCacheEnabled(BLOCK_CACHE) + .setTimeToLive(TTL) + .build()).build(); + admin.createTable(tableDesc, splitKeys).join(); + + assertEquals(NUM_REGIONS, TEST_UTIL.getHBaseCluster().getRegions(tableName).size()); + assertTrue("Table should be created with splitKyes + 1 rows in META", + admin.isTableAvailable(tableName, splitKeys).get()); + + // Clone & Verify + admin.cloneTableSchema(tableName, newTableName, preserveSplits).join(); + TableDescriptor newTableDesc = admin.getDescriptor(newTableName).get(); + + assertEquals(NUM_FAMILYS, newTableDesc.getColumnFamilyCount()); + assertEquals(BLOCK_SIZE, newTableDesc.getColumnFamily(FAMILY_1).getBlocksize()); + assertEquals(BLOCK_CACHE, newTableDesc.getColumnFamily(FAMILY_1).isBlockCacheEnabled()); + assertEquals(TTL, newTableDesc.getColumnFamily(FAMILY_1).getTimeToLive()); + TEST_UTIL.verifyTableDescriptorIgnoreTableName(tableDesc, newTableDesc); + + if (preserveSplits) { + assertEquals(NUM_REGIONS, TEST_UTIL.getHBaseCluster().getRegions(newTableName).size()); + assertTrue("New table should be created with splitKyes + 1 rows in META", + admin.isTableAvailable(newTableName, splitKeys).get()); + } else { + assertEquals(1, TEST_UTIL.getHBaseCluster().getRegions(newTableName).size()); + } + } + + @Test + public void testCloneTableSchemaWithNonExistentSourceTable() throws Exception { + final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new"); + // test for non-existent source table + try { + admin.cloneTableSchema(tableName, newTableName, false).join(); + fail("Should have failed when source table doesn't exist."); + } catch (CompletionException e) { + assertTrue(e.getCause() instanceof TableNotFoundException); + } + } + + @Test + public void testCloneTableSchemaWithExistentDestinationTable() throws Exception { + final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new"); + byte[] FAMILY_0 = Bytes.toBytes("cf0"); + TEST_UTIL.createTable(tableName, FAMILY_0); + TEST_UTIL.createTable(newTableName, FAMILY_0); + // test for existent destination table + try { + admin.cloneTableSchema(tableName, newTableName, false).join(); + fail("Should have failed when destination table exists."); + } catch (CompletionException e) { + assertTrue(e.getCause() instanceof TableExistsException); + } + } + + @Test + public void testIsTableAvailableWithInexistantTable() throws Exception { + final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new"); + // test for inexistant table + assertFalse(admin.isTableAvailable(newTableName).get()); + } +} \ No newline at end of file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotTemporaryDirectory.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotTemporaryDirectory.java index 3cbf88df6421..7c584ac63c3e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotTemporaryDirectory.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotTemporaryDirectory.java @@ -159,9 +159,7 @@ public void testRestoreDisabledSnapshot() TableName tableName = TableName.valueOf("testtb-" + tid); byte[] emptySnapshot = Bytes.toBytes("emptySnaptb-" + tid); byte[] snapshotName0 = Bytes.toBytes("snaptb0-" + tid); - byte[] snapshotName1 = Bytes.toBytes("snaptb1-" + tid); int snapshot0Rows; - int snapshot1Rows; // create Table and disable it SnapshotTestingUtils.createTable(UTIL, tableName, getNumReplicas(), TEST_FAM); @@ -177,21 +175,9 @@ public void testRestoreDisabledSnapshot() snapshot0Rows = UTIL.countRows(table); } admin.disableTable(tableName); - // take a snapshot takeSnapshot(tableName, Bytes.toString(snapshotName0), true); - // enable table and insert more data - admin.enableTable(tableName); - SnapshotTestingUtils.loadData(UTIL, tableName, 500, TEST_FAM); - try (Table table = UTIL.getConnection().getTable(tableName)) { - snapshot1Rows = UTIL.countRows(table); - } - - SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshot1Rows); - admin.disableTable(tableName); - takeSnapshot(tableName, Bytes.toString(snapshotName1), true); - // Restore from snapshot-0 admin.restoreSnapshot(snapshotName0); admin.enableTable(tableName); @@ -204,19 +190,6 @@ public void testRestoreDisabledSnapshot() admin.enableTable(tableName); SnapshotTestingUtils.verifyRowCount(UTIL, tableName, 0); SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas()); - - // Restore from snapshot-1 - admin.disableTable(tableName); - admin.restoreSnapshot(snapshotName1); - admin.enableTable(tableName); - SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshot1Rows); - SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas()); - - // Restore from snapshot-1 - UTIL.deleteTable(tableName); - admin.restoreSnapshot(snapshotName1); - SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshot1Rows); - SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas()); } @Test @@ -226,9 +199,7 @@ public void testRestoreEnabledSnapshot() TableName tableName = TableName.valueOf("testtb-" + tid); byte[] emptySnapshot = Bytes.toBytes("emptySnaptb-" + tid); byte[] snapshotName0 = Bytes.toBytes("snaptb0-" + tid); - byte[] snapshotName1 = Bytes.toBytes("snaptb1-" + tid); int snapshot0Rows; - int snapshot1Rows; // create Table SnapshotTestingUtils.createTable(UTIL, tableName, getNumReplicas(), TEST_FAM); @@ -241,19 +212,9 @@ public void testRestoreEnabledSnapshot() try (Table table = UTIL.getConnection().getTable(tableName)) { snapshot0Rows = UTIL.countRows(table); } - // take a snapshot takeSnapshot(tableName, Bytes.toString(snapshotName0), false); - // Insert more data - SnapshotTestingUtils.loadData(UTIL, tableName, 500, TEST_FAM); - try (Table table = UTIL.getConnection().getTable(tableName)) { - snapshot1Rows = UTIL.countRows(table); - } - - SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshot1Rows); - takeSnapshot(tableName, Bytes.toString(snapshotName1), false); - // Restore from snapshot-0 admin.disableTable(tableName); admin.restoreSnapshot(snapshotName0); @@ -267,19 +228,6 @@ public void testRestoreEnabledSnapshot() admin.enableTable(tableName); SnapshotTestingUtils.verifyRowCount(UTIL, tableName, 0); SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas()); - - // Restore from snapshot-1 - admin.disableTable(tableName); - admin.restoreSnapshot(snapshotName1); - admin.enableTable(tableName); - SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshot1Rows); - SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas()); - - // Restore from snapshot-1 - UTIL.deleteTable(tableName); - admin.restoreSnapshot(snapshotName1); - SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshot1Rows); - SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas()); } /** diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSCPBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSCPBase.java index aa5e4be47270..a37cac524dc5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSCPBase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSCPBase.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.master.assignment.AssignmentTestingUtil; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.util.Bytes; import org.junit.After; import org.junit.Before; import org.slf4j.Logger; @@ -138,9 +139,10 @@ protected long getSCPProcId(ProcedureExecutor procExec) { } protected Table createTable(final TableName tableName) throws IOException { - final Table t = this.util.createTable(tableName, HBaseTestingUtility.COLUMNS, - HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE, getRegionReplication()); - return t; + int numRegions = 10; + byte[][] splitKeys = Bytes.split(Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), numRegions - 3); + return util + .createTable(tableName, HBaseTestingUtility.COLUMNS, splitKeys, getRegionReplication()); } protected int getRegionReplication() { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicaFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicaFailover.java index 8a2123e9dbc4..205abf9f6c3e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicaFailover.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicaFailover.java @@ -324,7 +324,7 @@ public void run() { */ @Test public void testLotsOfRegionReplicas() throws IOException { - int numRegions = NB_SERVERS * 20; + int numRegions = NB_SERVERS * 5; int regionReplication = 10; String tableName = htd.getTableName().getNameAsString() + "2"; htd = HTU.createTableDescriptor(tableName); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFiles.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFiles.java index 8ea06e4e7f8a..781872e79d8a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFiles.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFiles.java @@ -21,24 +21,19 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; + import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Locale; -import java.util.Map; import java.util.TreeMap; import java.util.concurrent.atomic.AtomicInteger; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; @@ -47,8 +42,6 @@ import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; -import org.apache.hadoop.hbase.codec.KeyValueCodecWithTags; -import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; @@ -57,65 +50,22 @@ import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.HFileTestUtil; -import org.junit.AfterClass; -import org.junit.BeforeClass; import org.junit.ClassRule; -import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; -import org.apache.hbase.thirdparty.com.google.common.collect.Lists; /** * Test cases for the "load" half of the HFileOutputFormat bulk load functionality. These tests run * faster than the full MR cluster tests in TestHFileOutputFormat */ @Category({ MiscTests.class, LargeTests.class }) -public class TestLoadIncrementalHFiles { +public class TestLoadIncrementalHFiles extends TestLoadIncrementalHFilesBase { @ClassRule public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestLoadIncrementalHFiles.class); - @Rule - public TestName tn = new TestName(); - - private static final byte[] QUALIFIER = Bytes.toBytes("myqual"); - private static final byte[] FAMILY = Bytes.toBytes("myfam"); - private static final String NAMESPACE = "bulkNS"; - - static final String EXPECTED_MSG_FOR_NON_EXISTING_FAMILY = "Unmatched family names found"; - static final int MAX_FILES_PER_REGION_PER_FAMILY = 4; - - private static final byte[][] SPLIT_KEYS = - new byte[][] { Bytes.toBytes("ddd"), Bytes.toBytes("ppp") }; - - static HBaseTestingUtility util = new HBaseTestingUtility(); - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - util.getConfiguration().set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, ""); - util.getConfiguration().setInt(LoadIncrementalHFiles.MAX_FILES_PER_REGION_PER_FAMILY, - MAX_FILES_PER_REGION_PER_FAMILY); - // change default behavior so that tag values are returned with normal rpcs - util.getConfiguration().set(HConstants.RPC_CODEC_CONF_KEY, - KeyValueCodecWithTags.class.getCanonicalName()); - util.startMiniCluster(); - - setupNamespace(); - } - - protected static void setupNamespace() throws Exception { - util.getAdmin().createNamespace(NamespaceDescriptor.create(NAMESPACE).build()); - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { - util.shutdownMiniCluster(); - } - @Test public void testSimpleLoadWithMap() throws Exception { runTest("testSimpleLoadWithMap", BloomType.NONE, @@ -144,36 +94,6 @@ public void testSimpleLoadWithFileCopy() throws Exception { false, true, 2); } - /** - * Test case that creates some regions and loads HFiles that cross the boundaries of those regions - */ - @Test - public void testRegionCrossingLoad() throws Exception { - runTest("testRegionCrossingLoad", BloomType.NONE, - new byte[][][] { new byte[][] { Bytes.toBytes("aaaa"), Bytes.toBytes("eee") }, - new byte[][] { Bytes.toBytes("fff"), Bytes.toBytes("zzz") }, }); - } - - /** - * Test loading into a column family that has a ROW bloom filter. - */ - @Test - public void testRegionCrossingRowBloom() throws Exception { - runTest("testRegionCrossingLoadRowBloom", BloomType.ROW, - new byte[][][] { new byte[][] { Bytes.toBytes("aaaa"), Bytes.toBytes("eee") }, - new byte[][] { Bytes.toBytes("fff"), Bytes.toBytes("zzz") }, }); - } - - /** - * Test loading into a column family that has a ROWCOL bloom filter. - */ - @Test - public void testRegionCrossingRowColBloom() throws Exception { - runTest("testRegionCrossingLoadRowColBloom", BloomType.ROWCOL, - new byte[][][] { new byte[][] { Bytes.toBytes("aaaa"), Bytes.toBytes("eee") }, - new byte[][] { Bytes.toBytes("fff"), Bytes.toBytes("zzz") }, }); - } - /** * Test case that creates some regions and loads HFiles that have different region boundaries than * the table pre-split. @@ -187,33 +107,6 @@ public void testSimpleHFileSplit() throws Exception { new byte[][] { Bytes.toBytes("mmm"), Bytes.toBytes("zzz") }, }); } - /** - * Test case that creates some regions and loads HFiles that cross the boundaries and have - * different region boundaries than the table pre-split. - */ - @Test - public void testRegionCrossingHFileSplit() throws Exception { - testRegionCrossingHFileSplit(BloomType.NONE); - } - - /** - * Test case that creates some regions and loads HFiles that cross the boundaries have a ROW bloom - * filter and a different region boundaries than the table pre-split. - */ - @Test - public void testRegionCrossingHFileSplitRowBloom() throws Exception { - testRegionCrossingHFileSplit(BloomType.ROW); - } - - /** - * Test case that creates some regions and loads HFiles that cross the boundaries have a ROWCOL - * bloom filter and a different region boundaries than the table pre-split. - */ - @Test - public void testRegionCrossingHFileSplitRowColBloom() throws Exception { - testRegionCrossingHFileSplit(BloomType.ROWCOL); - } - @Test public void testSplitALot() throws Exception { runTest("testSplitALot", BloomType.NONE, @@ -226,187 +119,6 @@ public void testSplitALot() throws Exception { new byte[][][] { new byte[][] { Bytes.toBytes("aaaa"), Bytes.toBytes("zzz") }, }); } - private void testRegionCrossingHFileSplit(BloomType bloomType) throws Exception { - runTest("testHFileSplit" + bloomType + "Bloom", bloomType, - new byte[][] { Bytes.toBytes("aaa"), Bytes.toBytes("fff"), Bytes.toBytes("jjj"), - Bytes.toBytes("ppp"), Bytes.toBytes("uuu"), Bytes.toBytes("zzz"), }, - new byte[][][] { new byte[][] { Bytes.toBytes("aaaa"), Bytes.toBytes("eee") }, - new byte[][] { Bytes.toBytes("fff"), Bytes.toBytes("zzz") }, }); - } - - private TableDescriptor buildHTD(TableName tableName, BloomType bloomType) { - return TableDescriptorBuilder.newBuilder(tableName) - .setColumnFamily( - ColumnFamilyDescriptorBuilder.newBuilder(FAMILY).setBloomFilterType(bloomType).build()) - .build(); - } - - private void runTest(String testName, BloomType bloomType, byte[][][] hfileRanges) - throws Exception { - runTest(testName, bloomType, null, hfileRanges); - } - - private void runTest(String testName, BloomType bloomType, byte[][][] hfileRanges, boolean useMap) - throws Exception { - runTest(testName, bloomType, null, hfileRanges, useMap); - } - - private void runTest(String testName, BloomType bloomType, byte[][] tableSplitKeys, - byte[][][] hfileRanges) throws Exception { - runTest(testName, bloomType, tableSplitKeys, hfileRanges, false); - } - - private void runTest(String testName, BloomType bloomType, byte[][] tableSplitKeys, - byte[][][] hfileRanges, boolean useMap) throws Exception { - final byte[] TABLE_NAME = Bytes.toBytes("mytable_" + testName); - final boolean preCreateTable = tableSplitKeys != null; - - // Run the test bulkloading the table to the default namespace - final TableName TABLE_WITHOUT_NS = TableName.valueOf(TABLE_NAME); - runTest(testName, TABLE_WITHOUT_NS, bloomType, preCreateTable, tableSplitKeys, hfileRanges, - useMap, 2); - - - /* Run the test bulkloading the table from a depth of 3 - directory structure is now - baseDirectory - -- regionDir - -- familyDir - -- storeFileDir - */ - if (preCreateTable) { - runTest(testName + 2, TABLE_WITHOUT_NS, bloomType, true, tableSplitKeys, hfileRanges, - false, 3); - } - - // Run the test bulkloading the table to the specified namespace - final TableName TABLE_WITH_NS = TableName.valueOf(Bytes.toBytes(NAMESPACE), TABLE_NAME); - runTest(testName, TABLE_WITH_NS, bloomType, preCreateTable, tableSplitKeys, hfileRanges, - useMap, 2); - } - - private void runTest(String testName, TableName tableName, BloomType bloomType, - boolean preCreateTable, byte[][] tableSplitKeys, byte[][][] hfileRanges, - boolean useMap, int depth) throws Exception { - TableDescriptor htd = buildHTD(tableName, bloomType); - runTest(testName, htd, preCreateTable, tableSplitKeys, hfileRanges, useMap, false, depth); - } - - public static int loadHFiles(String testName, TableDescriptor htd, HBaseTestingUtility util, - byte[] fam, byte[] qual, boolean preCreateTable, byte[][] tableSplitKeys, - byte[][][] hfileRanges, boolean useMap, boolean deleteFile, boolean copyFiles, - int initRowCount, int factor) throws Exception { - return loadHFiles(testName, htd, util, fam, qual, preCreateTable, tableSplitKeys, hfileRanges, - useMap, deleteFile, copyFiles, initRowCount, factor, 2); - } - - public static int loadHFiles(String testName, TableDescriptor htd, HBaseTestingUtility util, - byte[] fam, byte[] qual, boolean preCreateTable, byte[][] tableSplitKeys, - byte[][][] hfileRanges, boolean useMap, boolean deleteFile, boolean copyFiles, - int initRowCount, int factor, int depth) throws Exception { - Path baseDirectory = util.getDataTestDirOnTestFS(testName); - FileSystem fs = util.getTestFileSystem(); - baseDirectory = baseDirectory.makeQualified(fs.getUri(), fs.getWorkingDirectory()); - Path parentDir = baseDirectory; - if (depth == 3) { - assert !useMap; - parentDir = new Path(baseDirectory, "someRegion"); - } - Path familyDir = new Path(parentDir, Bytes.toString(fam)); - - int hfileIdx = 0; - Map> map = null; - List list = null; - if (useMap || copyFiles) { - list = new ArrayList<>(); - } - if (useMap) { - map = new TreeMap<>(Bytes.BYTES_COMPARATOR); - map.put(fam, list); - } - Path last = null; - for (byte[][] range : hfileRanges) { - byte[] from = range[0]; - byte[] to = range[1]; - Path path = new Path(familyDir, "hfile_" + hfileIdx++); - HFileTestUtil.createHFile(util.getConfiguration(), fs, path, fam, qual, from, to, factor); - if (useMap) { - last = path; - list.add(path); - } - } - int expectedRows = hfileIdx * factor; - - TableName tableName = htd.getTableName(); - if (!util.getAdmin().tableExists(tableName) && (preCreateTable || map != null)) { - util.getAdmin().createTable(htd, tableSplitKeys); - } - - Configuration conf = util.getConfiguration(); - if (copyFiles) { - conf.setBoolean(LoadIncrementalHFiles.ALWAYS_COPY_FILES, true); - } - BulkLoadHFilesTool loader = new BulkLoadHFilesTool(conf); - List args = Lists.newArrayList(baseDirectory.toString(), tableName.toString()); - if (depth == 3) { - args.add("-loadTable"); - } - - if (useMap) { - if (deleteFile) { - fs.delete(last, true); - } - Map loaded = loader.bulkLoad(tableName, map); - if (deleteFile) { - expectedRows -= 1000; - for (BulkLoadHFiles.LoadQueueItem item : loaded.keySet()) { - if (item.getFilePath().getName().equals(last.getName())) { - fail(last + " should be missing"); - } - } - } - } else { - loader.run(args.toArray(new String[] {})); - } - - if (copyFiles) { - for (Path p : list) { - assertTrue(p + " should exist", fs.exists(p)); - } - } - - Table table = util.getConnection().getTable(tableName); - try { - assertEquals(initRowCount + expectedRows, util.countRows(table)); - } finally { - table.close(); - } - - return expectedRows; - } - - private void runTest(String testName, TableDescriptor htd, - boolean preCreateTable, byte[][] tableSplitKeys, byte[][][] hfileRanges, boolean useMap, - boolean copyFiles, int depth) throws Exception { - loadHFiles(testName, htd, util, FAMILY, QUALIFIER, preCreateTable, tableSplitKeys, hfileRanges, - useMap, true, copyFiles, 0, 1000, depth); - - final TableName tableName = htd.getTableName(); - // verify staging folder has been cleaned up - Path stagingBasePath = - new Path(FSUtils.getRootDir(util.getConfiguration()), HConstants.BULKLOAD_STAGING_DIR_NAME); - FileSystem fs = util.getTestFileSystem(); - if (fs.exists(stagingBasePath)) { - FileStatus[] files = fs.listStatus(stagingBasePath); - for (FileStatus file : files) { - assertTrue("Folder=" + file.getPath() + " is not cleaned up.", - file.getPath().getName() != "DONOTERASE"); - } - } - - util.deleteTable(tableName); - } - /** * Test that tags survive through a bulk load that needs to split hfiles. This test depends on the * "hbase.client.rpc.codec" = KeyValueCodecWithTags so that the client can get tags in the diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFiles2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFiles2.java new file mode 100644 index 000000000000..99cfe8b8cf90 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFiles2.java @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.tool; + +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.regionserver.BloomType; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Split from {@link TestLoadIncrementalHFiles}. + */ +@Category({ MiscTests.class, LargeTests.class }) +public class TestLoadIncrementalHFiles2 extends TestLoadIncrementalHFilesBase { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestLoadIncrementalHFiles2.class); + + /** + * Test case that creates some regions and loads HFiles that cross the boundaries and have + * different region boundaries than the table pre-split. + */ + @Test + public void testRegionCrossingHFileSplit() throws Exception { + testRegionCrossingHFileSplit(BloomType.NONE); + } + + /** + * Test case that creates some regions and loads HFiles that cross the boundaries have a ROW bloom + * filter and a different region boundaries than the table pre-split. + */ + @Test + public void testRegionCrossingHFileSplitRowBloom() throws Exception { + testRegionCrossingHFileSplit(BloomType.ROW); + } + + /** + * Test case that creates some regions and loads HFiles that cross the boundaries have a ROWCOL + * bloom filter and a different region boundaries than the table pre-split. + */ + @Test + public void testRegionCrossingHFileSplitRowColBloom() throws Exception { + testRegionCrossingHFileSplit(BloomType.ROWCOL); + } + + private void testRegionCrossingHFileSplit(BloomType bloomType) throws Exception { + runTest("testHFileSplit" + bloomType + "Bloom", bloomType, + new byte[][] { Bytes.toBytes("aaa"), Bytes.toBytes("fff"), Bytes.toBytes("jjj"), + Bytes.toBytes("ppp"), Bytes.toBytes("uuu"), Bytes.toBytes("zzz"), }, + new byte[][][] { new byte[][] { Bytes.toBytes("aaaa"), Bytes.toBytes("eee") }, + new byte[][] { Bytes.toBytes("fff"), Bytes.toBytes("zzz") }, }); + } + + /** + * Test case that creates some regions and loads HFiles that cross the boundaries of those regions + */ + @Test + public void testRegionCrossingLoad() throws Exception { + runTest("testRegionCrossingLoad", BloomType.NONE, + new byte[][][] { new byte[][] { Bytes.toBytes("aaaa"), Bytes.toBytes("eee") }, + new byte[][] { Bytes.toBytes("fff"), Bytes.toBytes("zzz") }, }); + } + + /** + * Test loading into a column family that has a ROW bloom filter. + */ + @Test + public void testRegionCrossingRowBloom() throws Exception { + runTest("testRegionCrossingLoadRowBloom", BloomType.ROW, + new byte[][][] { new byte[][] { Bytes.toBytes("aaaa"), Bytes.toBytes("eee") }, + new byte[][] { Bytes.toBytes("fff"), Bytes.toBytes("zzz") }, }); + } + + /** + * Test loading into a column family that has a ROWCOL bloom filter. + */ + @Test + public void testRegionCrossingRowColBloom() throws Exception { + runTest("testRegionCrossingLoadRowColBloom", BloomType.ROWCOL, + new byte[][][] { new byte[][] { Bytes.toBytes("aaaa"), Bytes.toBytes("eee") }, + new byte[][] { Bytes.toBytes("fff"), Bytes.toBytes("zzz") }, }); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFilesBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFilesBase.java new file mode 100644 index 000000000000..82dac05ebeaa --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFilesBase.java @@ -0,0 +1,284 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.tool; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.NamespaceDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.codec.KeyValueCodecWithTags; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.regionserver.BloomType; +import org.apache.hadoop.hbase.security.HadoopSecurityEnabledUserProviderForTesting; +import org.apache.hadoop.hbase.security.UserProvider; +import org.apache.hadoop.hbase.security.access.AccessControlLists; +import org.apache.hadoop.hbase.security.access.SecureTestUtil; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.HFileTestUtil; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.rules.TestName; + +import org.apache.hbase.thirdparty.com.google.common.collect.Lists; + +/** + * Base class for TestLoadIncrementalHFiles. + */ +public class TestLoadIncrementalHFilesBase { + @Rule + public TestName tn = new TestName(); + + protected static final byte[] QUALIFIER = Bytes.toBytes("myqual"); + protected static final byte[] FAMILY = Bytes.toBytes("myfam"); + private static final String NAMESPACE = "bulkNS"; + + static final String EXPECTED_MSG_FOR_NON_EXISTING_FAMILY = "Unmatched family names found"; + static final int MAX_FILES_PER_REGION_PER_FAMILY = 4; + + protected static final byte[][] SPLIT_KEYS = + new byte[][] { Bytes.toBytes("ddd"), Bytes.toBytes("ppp") }; + + static HBaseTestingUtility util = new HBaseTestingUtility(); + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + util.getConfiguration().set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, ""); + util.getConfiguration().setInt(LoadIncrementalHFiles.MAX_FILES_PER_REGION_PER_FAMILY, + MAX_FILES_PER_REGION_PER_FAMILY); + // change default behavior so that tag values are returned with normal rpcs + util.getConfiguration().set(HConstants.RPC_CODEC_CONF_KEY, + KeyValueCodecWithTags.class.getCanonicalName()); + util.startMiniCluster(); + + setupNamespace(); + } + + protected static void setupNamespace() throws Exception { + util.getAdmin().createNamespace(NamespaceDescriptor.create(NAMESPACE).build()); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + util.shutdownMiniCluster(); + } + + public static void secureSetUpBeforeClass() throws Exception { + // set the always on security provider + UserProvider.setUserProviderForTesting(util.getConfiguration(), + HadoopSecurityEnabledUserProviderForTesting.class); + // setup configuration + SecureTestUtil.enableSecurity(util.getConfiguration()); + util.getConfiguration().setInt(LoadIncrementalHFiles.MAX_FILES_PER_REGION_PER_FAMILY, + MAX_FILES_PER_REGION_PER_FAMILY); + // change default behavior so that tag values are returned with normal rpcs + util.getConfiguration().set(HConstants.RPC_CODEC_CONF_KEY, + KeyValueCodecWithTags.class.getCanonicalName()); + + util.startMiniCluster(); + + // Wait for the ACL table to become available + util.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME); + + setupNamespace(); + } + + protected void runTest(String testName, BloomType bloomType, byte[][][] hfileRanges) + throws Exception { + runTest(testName, bloomType, null, hfileRanges); + } + + protected void runTest(String testName, BloomType bloomType, byte[][][] hfileRanges, + boolean useMap) throws Exception { + runTest(testName, bloomType, null, hfileRanges, useMap); + } + + protected void runTest(String testName, BloomType bloomType, byte[][] tableSplitKeys, + byte[][][] hfileRanges) throws Exception { + runTest(testName, bloomType, tableSplitKeys, hfileRanges, false); + } + + protected void runTest(String testName, BloomType bloomType, byte[][] tableSplitKeys, + byte[][][] hfileRanges, boolean useMap) throws Exception { + final byte[] TABLE_NAME = Bytes.toBytes("mytable_" + testName); + final boolean preCreateTable = tableSplitKeys != null; + + // Run the test bulkloading the table to the default namespace + final TableName TABLE_WITHOUT_NS = TableName.valueOf(TABLE_NAME); + runTest(testName, TABLE_WITHOUT_NS, bloomType, preCreateTable, tableSplitKeys, hfileRanges, + useMap, 2); + + + /* Run the test bulkloading the table from a depth of 3 + directory structure is now + baseDirectory + -- regionDir + -- familyDir + -- storeFileDir + */ + if (preCreateTable) { + runTest(testName + 2, TABLE_WITHOUT_NS, bloomType, true, tableSplitKeys, hfileRanges, + false, 3); + } + + // Run the test bulkloading the table to the specified namespace + final TableName TABLE_WITH_NS = TableName.valueOf(Bytes.toBytes(NAMESPACE), TABLE_NAME); + runTest(testName, TABLE_WITH_NS, bloomType, preCreateTable, tableSplitKeys, hfileRanges, + useMap, 2); + } + + protected void runTest(String testName, TableName tableName, BloomType bloomType, + boolean preCreateTable, byte[][] tableSplitKeys, byte[][][] hfileRanges, + boolean useMap, int depth) throws Exception { + TableDescriptor htd = buildHTD(tableName, bloomType); + runTest(testName, htd, preCreateTable, tableSplitKeys, hfileRanges, useMap, false, depth); + } + + protected void runTest(String testName, TableDescriptor htd, + boolean preCreateTable, byte[][] tableSplitKeys, byte[][][] hfileRanges, boolean useMap, + boolean copyFiles, int depth) throws Exception { + loadHFiles(testName, htd, util, FAMILY, QUALIFIER, preCreateTable, tableSplitKeys, hfileRanges, + useMap, true, copyFiles, 0, 1000, depth); + + final TableName tableName = htd.getTableName(); + // verify staging folder has been cleaned up + Path stagingBasePath = + new Path(FSUtils.getRootDir(util.getConfiguration()), HConstants.BULKLOAD_STAGING_DIR_NAME); + FileSystem fs = util.getTestFileSystem(); + if (fs.exists(stagingBasePath)) { + FileStatus[] files = fs.listStatus(stagingBasePath); + for (FileStatus file : files) { + assertTrue("Folder=" + file.getPath() + " is not cleaned up.", + file.getPath().getName() != "DONOTERASE"); + } + } + + util.deleteTable(tableName); + } + + protected TableDescriptor buildHTD(TableName tableName, BloomType bloomType) { + return TableDescriptorBuilder.newBuilder(tableName) + .setColumnFamily( + ColumnFamilyDescriptorBuilder.newBuilder(FAMILY).setBloomFilterType(bloomType).build()) + .build(); + } + + public static int loadHFiles(String testName, TableDescriptor htd, HBaseTestingUtility util, + byte[] fam, byte[] qual, boolean preCreateTable, byte[][] tableSplitKeys, + byte[][][] hfileRanges, boolean useMap, boolean deleteFile, boolean copyFiles, + int initRowCount, int factor, int depth) throws Exception { + Path baseDirectory = util.getDataTestDirOnTestFS(testName); + FileSystem fs = util.getTestFileSystem(); + baseDirectory = baseDirectory.makeQualified(fs.getUri(), fs.getWorkingDirectory()); + Path parentDir = baseDirectory; + if (depth == 3) { + assert !useMap; + parentDir = new Path(baseDirectory, "someRegion"); + } + Path familyDir = new Path(parentDir, Bytes.toString(fam)); + + int hfileIdx = 0; + Map> map = null; + List list = null; + if (useMap || copyFiles) { + list = new ArrayList<>(); + } + if (useMap) { + map = new TreeMap<>(Bytes.BYTES_COMPARATOR); + map.put(fam, list); + } + Path last = null; + for (byte[][] range : hfileRanges) { + byte[] from = range[0]; + byte[] to = range[1]; + Path path = new Path(familyDir, "hfile_" + hfileIdx++); + HFileTestUtil.createHFile(util.getConfiguration(), fs, path, fam, qual, from, to, factor); + if (useMap) { + last = path; + list.add(path); + } + } + int expectedRows = hfileIdx * factor; + + TableName tableName = htd.getTableName(); + if (!util.getAdmin().tableExists(tableName) && (preCreateTable || map != null)) { + util.getAdmin().createTable(htd, tableSplitKeys); + } + + Configuration conf = util.getConfiguration(); + if (copyFiles) { + conf.setBoolean(LoadIncrementalHFiles.ALWAYS_COPY_FILES, true); + } + BulkLoadHFilesTool loader = new BulkLoadHFilesTool(conf); + List args = Lists.newArrayList(baseDirectory.toString(), tableName.toString()); + if (depth == 3) { + args.add("-loadTable"); + } + + if (useMap) { + if (deleteFile) { + fs.delete(last, true); + } + Map loaded = loader.bulkLoad(tableName, map); + if (deleteFile) { + expectedRows -= 1000; + for (BulkLoadHFiles.LoadQueueItem item : loaded.keySet()) { + if (item.getFilePath().getName().equals(last.getName())) { + fail(last + " should be missing"); + } + } + } + } else { + loader.run(args.toArray(new String[] {})); + } + + if (copyFiles) { + for (Path p : list) { + assertTrue(p + " should exist", fs.exists(p)); + } + } + + Table table = util.getConnection().getTable(tableName); + try { + assertEquals(initRowCount + expectedRows, util.countRows(table)); + } finally { + table.close(); + } + + return expectedRows; + } +} \ No newline at end of file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestSecureLoadIncrementalHFiles.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestSecureLoadIncrementalHFiles.java index 4e10f01fa58d..60527839df54 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestSecureLoadIncrementalHFiles.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestSecureLoadIncrementalHFiles.java @@ -18,12 +18,6 @@ package org.apache.hadoop.hbase.tool; import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.codec.KeyValueCodecWithTags; -import org.apache.hadoop.hbase.security.HadoopSecurityEnabledUserProviderForTesting; -import org.apache.hadoop.hbase.security.UserProvider; -import org.apache.hadoop.hbase.security.access.AccessControlLists; -import org.apache.hadoop.hbase.security.access.SecureTestUtil; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.junit.BeforeClass; @@ -48,23 +42,6 @@ public class TestSecureLoadIncrementalHFiles extends TestLoadIncrementalHFiles { @BeforeClass public static void setUpBeforeClass() throws Exception { - // set the always on security provider - UserProvider.setUserProviderForTesting(util.getConfiguration(), - HadoopSecurityEnabledUserProviderForTesting.class); - // setup configuration - SecureTestUtil.enableSecurity(util.getConfiguration()); - util.getConfiguration().setInt(LoadIncrementalHFiles.MAX_FILES_PER_REGION_PER_FAMILY, - MAX_FILES_PER_REGION_PER_FAMILY); - // change default behavior so that tag values are returned with normal rpcs - util.getConfiguration().set(HConstants.RPC_CODEC_CONF_KEY, - KeyValueCodecWithTags.class.getCanonicalName()); - - util.startMiniCluster(); - - // Wait for the ACL table to become available - util.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME); - - setupNamespace(); + TestLoadIncrementalHFilesBase.secureSetUpBeforeClass(); } - -} +} \ No newline at end of file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestSecureLoadIncrementalHFiles2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestSecureLoadIncrementalHFiles2.java new file mode 100644 index 000000000000..99c545804fa5 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestSecureLoadIncrementalHFiles2.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.tool; + +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.experimental.categories.Category; + +/** + * Reruns TestLoadIncrementalHFiles using LoadIncrementalHFiles in secure mode. This suite is unable + * to verify the security handoff/turnover as miniCluster is running as system user thus has root + * privileges and delegation tokens don't seem to work on miniDFS. + *

+ * Thus SecureBulkload can only be completely verified by running integration tests against a secure + * cluster. This suite is still invaluable as it verifies the other mechanisms that need to be + * supported as part of a LoadIncrementalFiles call. + */ +@Category({ MiscTests.class, LargeTests.class }) +public class TestSecureLoadIncrementalHFiles2 extends TestLoadIncrementalHFiles2 { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestSecureLoadIncrementalHFiles2.class); + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TestLoadIncrementalHFilesBase.secureSetUpBeforeClass(); + } +} \ No newline at end of file diff --git a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestAdminShell3.java b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestAdminShell3.java new file mode 100644 index 000000000000..265985d7fa38 --- /dev/null +++ b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestAdminShell3.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import java.io.IOException; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.testclassification.ClientTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.jruby.embed.PathType; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ ClientTests.class, LargeTests.class }) +public class TestAdminShell3 extends AbstractTestShell { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestAdminShell3.class); + + @Test + public void testRunShellTests() throws IOException { + System.setProperty("shell.test.include", "admin3_test.rb"); + // Start all ruby tests + jruby.runScriptlet(PathType.ABSOLUTE, "src/test/ruby/tests_runner.rb"); + } +} diff --git a/hbase-shell/src/test/ruby/hbase/admin3_test.rb b/hbase-shell/src/test/ruby/hbase/admin3_test.rb new file mode 100644 index 000000000000..e1996a992973 --- /dev/null +++ b/hbase-shell/src/test/ruby/hbase/admin3_test.rb @@ -0,0 +1,298 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'shell' +require 'stringio' +require 'hbase_constants' +require 'hbase/hbase' +require 'hbase/table' + +include HBaseConstants + +module Hbase + # Simple administration methods tests + class AdminRegionTest < Test::Unit::TestCase + include TestHelpers + def setup + setup_hbase + # Create test table if it does not exist + @test_name = "hbase_shell_tests_table" + drop_test_table(@test_name) + create_test_table(@test_name) + end + + def teardown + shutdown + end + + define_test "unassign should allow encoded region names" do + region = command(:locate_region, @test_name, '') + regionName = region.getRegionInfo().getRegionNameAsString() + command(:unassign, regionName, true) + end + + define_test "unassign should allow non-encoded region names" do + region = command(:locate_region, @test_name, '') + encodedRegionName = region.getRegionInfo().getEncodedName() + command(:unassign, encodedRegionName, true) + end + + define_test "list regions should allow table name" do + command(:list_regions, @test_name) + end + + define_test 'merge regions' do + @t_name = 'hbase_shell_merge' + @t_name2 = 'hbase_shell_merge_2' + drop_test_table(@t_name) + drop_test_table(@t_name2) + admin.create(@t_name, 'a', NUMREGIONS => 10, SPLITALGO => 'HexStringSplit') + r1 = command(:locate_region, @t_name, '1') + r2 = command(:locate_region, @t_name, '2') + r3 = command(:locate_region, @t_name, '4') + r4 = command(:locate_region, @t_name, '5') + r5 = command(:locate_region, @t_name, '7') + r6 = command(:locate_region, @t_name, '8') + region1 = r1.getRegion.getRegionNameAsString + region2 = r2.getRegion.getRegionNameAsString + region3 = r3.getRegion.getRegionNameAsString + region4 = r4.getRegion.getRegionNameAsString + region5 = r5.getRegion.getRegionNameAsString + region6 = r6.getRegion.getRegionNameAsString + # only 1 region + assert_raise(ArgumentError) do + command(:merge_region, 'a') + end + # only 1 region with force=true + assert_raise(ArgumentError) do + command(:merge_region, 'a', true) + end + # non-existing region + assert_raise(RuntimeError) do + command(:merge_region, 'a','b') + end + # duplicate regions + assert_raise(RuntimeError) do + command(:merge_region, region1,region1,region1) + end + # 3 non-adjacent regions without forcible=true + assert_raise(RuntimeError) do + command(:merge_region, region1,region2,region4) + end + # 2 adjacent regions + command(:merge_region, region1,region2) + # 3 non-adjacent regions with forcible=true + command(:merge_region, region3,region5,region6, true) + + admin.create(@t_name2, 'a', NUMREGIONS => 5, SPLITALGO => 'HexStringSplit') + r1 = command(:locate_region, @t_name2, '1') + r2 = command(:locate_region, @t_name2, '4') + r3 = command(:locate_region, @t_name2, '7') + region1 = r1.getRegion.getRegionNameAsString + region2 = r2.getRegion.getRegionNameAsString + region3 = r3.getRegion.getRegionNameAsString + + # accept array of regions + command(:merge_region, [region1,region2,region3]) + end + end + + # Simple administration methods tests + # rubocop:disable Metrics/ClassLength + class AdminAlterTableTest < Test::Unit::TestCase + include TestHelpers + + def setup + setup_hbase + # Create test table if it does not exist + @test_name = "hbase_shell_tests_table" + drop_test_table(@test_name) + create_test_table(@test_name) + end + + def teardown + shutdown + end + + #------------------------------------------------------------------------------- + + define_test "alter should fail with non-string table names" do + assert_raise(ArgumentError) do + command(:alter, 123, METHOD => 'delete', NAME => 'y') + end + end + + define_test "alter should fail with non-existing tables" do + assert_raise(ArgumentError) do + command(:alter, 'NOT.EXISTS', METHOD => 'delete', NAME => 'y') + end + end + + define_test "alter should not fail with enabled tables" do + command(:enable, @test_name) + command(:alter, @test_name, METHOD => 'delete', NAME => 'y') + end + + define_test "alter should be able to delete column families" do + assert_equal(['x:', 'y:'], table(@test_name).get_all_columns.sort) + command(:alter, @test_name, METHOD => 'delete', NAME => 'y') + command(:enable, @test_name) + assert_equal(['x:'], table(@test_name).get_all_columns.sort) + end + + define_test "alter should be able to add column families" do + assert_equal(['x:', 'y:'], table(@test_name).get_all_columns.sort) + command(:alter, @test_name, NAME => 'z') + command(:enable, @test_name) + assert_equal(['x:', 'y:', 'z:'], table(@test_name).get_all_columns.sort) + end + + define_test "alter should be able to add column families (name-only alter spec)" do + assert_equal(['x:', 'y:'], table(@test_name).get_all_columns.sort) + command(:alter, @test_name, 'z') + command(:enable, @test_name) + assert_equal(['x:', 'y:', 'z:'], table(@test_name).get_all_columns.sort) + end + + define_test 'alter should support more than one alteration in one call' do + assert_equal(['x:', 'y:'], table(@test_name).get_all_columns.sort) + alter_out_put = capture_stdout do + command(:alter, @test_name, { NAME => 'z' }, + { METHOD => 'delete', NAME => 'y' }, + 'MAX_FILESIZE' => 12_345_678) + end + command(:enable, @test_name) + assert_equal(1, /Updating all regions/.match(alter_out_put).size, + "HBASE-15641 - Should only perform one table + modification per alter.") + assert_equal(['x:', 'z:'], table(@test_name).get_all_columns.sort) + assert_match(/12345678/, admin.describe(@test_name)) + end + + define_test 'alter should be able to set the TargetRegionSize and TargetRegionCount' do + command(:alter, @test_name, 'NORMALIZER_TARGET_REGION_COUNT' => 156) + assert_match(/156/, admin.describe(@test_name)) + command(:alter, @test_name, 'NORMALIZER_TARGET_REGION_SIZE' => 234) + assert_match(/234/, admin.describe(@test_name)) + end + + define_test 'alter should support shortcut DELETE alter specs' do + assert_equal(['x:', 'y:'], table(@test_name).get_all_columns.sort) + command(:alter, @test_name, 'delete' => 'y') + assert_equal(['x:'], table(@test_name).get_all_columns.sort) + end + + define_test "alter should be able to change table options" do + command(:alter, @test_name, METHOD => 'table_att', 'MAX_FILESIZE' => 12345678) + assert_match(/12345678/, admin.describe(@test_name)) + end + + define_test "alter should be able to change table options w/o table_att" do + command(:alter, @test_name, 'MAX_FILESIZE' => 12345678) + assert_match(/12345678/, admin.describe(@test_name)) + end + + define_test "alter should be able to change coprocessor attributes" do + drop_test_table(@test_name) + create_test_table(@test_name) + + cp_key = "coprocessor" + class_name = "org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver" + + cp_value = "|" + class_name + "|12|arg1=1,arg2=2" + + # eval() is used to convert a string to regex + assert_no_match(eval("/" + class_name + "/"), admin.describe(@test_name)) + assert_no_match(eval("/" + cp_key + "/"), admin.describe(@test_name)) + command(:alter, @test_name, 'METHOD' => 'table_att', cp_key => cp_value) + assert_match(eval("/" + class_name + "/"), admin.describe(@test_name)) + assert_match(eval("/" + cp_key + "\\$(\\d+)/"), admin.describe(@test_name)) + end + + define_test "alter should be able to remove a table attribute" do + drop_test_table(@test_name) + create_test_table(@test_name) + + key = "MAX_FILESIZE" + command(:alter, @test_name, 'METHOD' => 'table_att', key => 12345678) + + # eval() is used to convert a string to regex + assert_match(eval("/" + key + "/"), admin.describe(@test_name)) + + command(:alter, @test_name, 'METHOD' => 'table_att_unset', 'NAME' => key) + assert_no_match(eval("/" + key + "/"), admin.describe(@test_name)) + end + + define_test "alter should be able to remove a list of table attributes" do + drop_test_table(@test_name) + + key_1 = "TestAttr1" + key_2 = "TestAttr2" + command(:create, @test_name, { NAME => 'i'}, METADATA => { key_1 => 1, key_2 => 2 }) + + # eval() is used to convert a string to regex + assert_match(eval("/" + key_1 + "/"), admin.describe(@test_name)) + assert_match(eval("/" + key_2 + "/"), admin.describe(@test_name)) + + command(:alter, @test_name, 'METHOD' => 'table_att_unset', 'NAME' => [ key_1, key_2 ]) + assert_no_match(eval("/" + key_1 + "/"), admin.describe(@test_name)) + assert_no_match(eval("/" + key_2 + "/"), admin.describe(@test_name)) + end + + define_test "alter should be able to remove a table configuration" do + drop_test_table(@test_name) + create_test_table(@test_name) + + key = "TestConf" + command(:alter, @test_name, CONFIGURATION => {key => 1}) + + # eval() is used to convert a string to regex + assert_match(eval("/" + key + "/"), admin.describe(@test_name)) + + command(:alter, @test_name, 'METHOD' => 'table_conf_unset', 'NAME' => key) + assert_no_match(eval("/" + key + "/"), admin.describe(@test_name)) + end + + define_test "alter should be able to remove a list of table configuration" do + drop_test_table(@test_name) + + key_1 = "TestConf1" + key_2 = "TestConf2" + command(:create, @test_name, { NAME => 'i'}, CONFIGURATION => { key_1 => 1, key_2 => 2 }) + + # eval() is used to convert a string to regex + assert_match(eval("/" + key_1 + "/"), admin.describe(@test_name)) + assert_match(eval("/" + key_2 + "/"), admin.describe(@test_name)) + + command(:alter, @test_name, 'METHOD' => 'table_conf_unset', 'NAME' => [ key_1, key_2 ]) + assert_no_match(eval("/" + key_1 + "/"), admin.describe(@test_name)) + assert_no_match(eval("/" + key_2 + "/"), admin.describe(@test_name)) + end + + define_test "get_table should get a real table" do + drop_test_table(@test_name) + create_test_table(@test_name) + + table = table(@test_name) + assert_not_equal(nil, table) + table.close + end + end + # rubocop:enable Metrics/ClassLength +end diff --git a/hbase-shell/src/test/ruby/hbase/admin_test.rb b/hbase-shell/src/test/ruby/hbase/admin_test.rb index 8499fe37972f..7d4921f0c465 100644 --- a/hbase-shell/src/test/ruby/hbase/admin_test.rb +++ b/hbase-shell/src/test/ruby/hbase/admin_test.rb @@ -528,274 +528,5 @@ def teardown end end end - - # Simple administration methods tests - class AdminRegionTest < Test::Unit::TestCase - include TestHelpers - def setup - setup_hbase - # Create test table if it does not exist - @test_name = "hbase_shell_tests_table" - drop_test_table(@test_name) - create_test_table(@test_name) - end - - def teardown - shutdown - end - - define_test "unassign should allow encoded region names" do - region = command(:locate_region, @test_name, '') - regionName = region.getRegionInfo().getRegionNameAsString() - command(:unassign, regionName, true) - end - - define_test "unassign should allow non-encoded region names" do - region = command(:locate_region, @test_name, '') - encodedRegionName = region.getRegionInfo().getEncodedName() - command(:unassign, encodedRegionName, true) - end - - define_test "list regions should allow table name" do - command(:list_regions, @test_name) - end - - define_test 'merge regions' do - @t_name = 'hbase_shell_merge' - @t_name2 = 'hbase_shell_merge_2' - drop_test_table(@t_name) - drop_test_table(@t_name2) - admin.create(@t_name, 'a', NUMREGIONS => 10, SPLITALGO => 'HexStringSplit') - r1 = command(:locate_region, @t_name, '1') - r2 = command(:locate_region, @t_name, '2') - r3 = command(:locate_region, @t_name, '4') - r4 = command(:locate_region, @t_name, '5') - r5 = command(:locate_region, @t_name, '7') - r6 = command(:locate_region, @t_name, '8') - region1 = r1.getRegion.getRegionNameAsString - region2 = r2.getRegion.getRegionNameAsString - region3 = r3.getRegion.getRegionNameAsString - region4 = r4.getRegion.getRegionNameAsString - region5 = r5.getRegion.getRegionNameAsString - region6 = r6.getRegion.getRegionNameAsString - # only 1 region - assert_raise(ArgumentError) do - command(:merge_region, 'a') - end - # only 1 region with force=true - assert_raise(ArgumentError) do - command(:merge_region, 'a', true) - end - # non-existing region - assert_raise(RuntimeError) do - command(:merge_region, 'a','b') - end - # duplicate regions - assert_raise(RuntimeError) do - command(:merge_region, region1,region1,region1) - end - # 3 non-adjacent regions without forcible=true - assert_raise(RuntimeError) do - command(:merge_region, region1,region2,region4) - end - # 2 adjacent regions - command(:merge_region, region1,region2) - # 3 non-adjacent regions with forcible=true - command(:merge_region, region3,region5,region6, true) - - admin.create(@t_name2, 'a', NUMREGIONS => 5, SPLITALGO => 'HexStringSplit') - r1 = command(:locate_region, @t_name2, '1') - r2 = command(:locate_region, @t_name2, '4') - r3 = command(:locate_region, @t_name2, '7') - region1 = r1.getRegion.getRegionNameAsString - region2 = r2.getRegion.getRegionNameAsString - region3 = r3.getRegion.getRegionNameAsString - - # accept array of regions - command(:merge_region, [region1,region2,region3]) - end - end - - # Simple administration methods tests - # rubocop:disable Metrics/ClassLength - class AdminAlterTableTest < Test::Unit::TestCase - include TestHelpers - - def setup - setup_hbase - # Create test table if it does not exist - @test_name = "hbase_shell_tests_table" - drop_test_table(@test_name) - create_test_table(@test_name) - end - - def teardown - shutdown - end - - #------------------------------------------------------------------------------- - - define_test "alter should fail with non-string table names" do - assert_raise(ArgumentError) do - command(:alter, 123, METHOD => 'delete', NAME => 'y') - end - end - - define_test "alter should fail with non-existing tables" do - assert_raise(ArgumentError) do - command(:alter, 'NOT.EXISTS', METHOD => 'delete', NAME => 'y') - end - end - - define_test "alter should not fail with enabled tables" do - command(:enable, @test_name) - command(:alter, @test_name, METHOD => 'delete', NAME => 'y') - end - - define_test "alter should be able to delete column families" do - assert_equal(['x:', 'y:'], table(@test_name).get_all_columns.sort) - command(:alter, @test_name, METHOD => 'delete', NAME => 'y') - command(:enable, @test_name) - assert_equal(['x:'], table(@test_name).get_all_columns.sort) - end - - define_test "alter should be able to add column families" do - assert_equal(['x:', 'y:'], table(@test_name).get_all_columns.sort) - command(:alter, @test_name, NAME => 'z') - command(:enable, @test_name) - assert_equal(['x:', 'y:', 'z:'], table(@test_name).get_all_columns.sort) - end - - define_test "alter should be able to add column families (name-only alter spec)" do - assert_equal(['x:', 'y:'], table(@test_name).get_all_columns.sort) - command(:alter, @test_name, 'z') - command(:enable, @test_name) - assert_equal(['x:', 'y:', 'z:'], table(@test_name).get_all_columns.sort) - end - - define_test 'alter should support more than one alteration in one call' do - assert_equal(['x:', 'y:'], table(@test_name).get_all_columns.sort) - alter_out_put = capture_stdout do - command(:alter, @test_name, { NAME => 'z' }, - { METHOD => 'delete', NAME => 'y' }, - 'MAX_FILESIZE' => 12_345_678) - end - command(:enable, @test_name) - assert_equal(1, /Updating all regions/.match(alter_out_put).size, - "HBASE-15641 - Should only perform one table - modification per alter.") - assert_equal(['x:', 'z:'], table(@test_name).get_all_columns.sort) - assert_match(/12345678/, admin.describe(@test_name)) - end - - define_test 'alter should be able to set the TargetRegionSize and TargetRegionCount' do - command(:alter, @test_name, 'NORMALIZER_TARGET_REGION_COUNT' => 156) - assert_match(/156/, admin.describe(@test_name)) - command(:alter, @test_name, 'NORMALIZER_TARGET_REGION_SIZE' => 234) - assert_match(/234/, admin.describe(@test_name)) - end - - define_test 'alter should support shortcut DELETE alter specs' do - assert_equal(['x:', 'y:'], table(@test_name).get_all_columns.sort) - command(:alter, @test_name, 'delete' => 'y') - assert_equal(['x:'], table(@test_name).get_all_columns.sort) - end - - define_test "alter should be able to change table options" do - command(:alter, @test_name, METHOD => 'table_att', 'MAX_FILESIZE' => 12345678) - assert_match(/12345678/, admin.describe(@test_name)) - end - - define_test "alter should be able to change table options w/o table_att" do - command(:alter, @test_name, 'MAX_FILESIZE' => 12345678) - assert_match(/12345678/, admin.describe(@test_name)) - end - - define_test "alter should be able to change coprocessor attributes" do - drop_test_table(@test_name) - create_test_table(@test_name) - - cp_key = "coprocessor" - class_name = "org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver" - - cp_value = "|" + class_name + "|12|arg1=1,arg2=2" - - # eval() is used to convert a string to regex - assert_no_match(eval("/" + class_name + "/"), admin.describe(@test_name)) - assert_no_match(eval("/" + cp_key + "/"), admin.describe(@test_name)) - command(:alter, @test_name, 'METHOD' => 'table_att', cp_key => cp_value) - assert_match(eval("/" + class_name + "/"), admin.describe(@test_name)) - assert_match(eval("/" + cp_key + "\\$(\\d+)/"), admin.describe(@test_name)) - end - - define_test "alter should be able to remove a table attribute" do - drop_test_table(@test_name) - create_test_table(@test_name) - - key = "MAX_FILESIZE" - command(:alter, @test_name, 'METHOD' => 'table_att', key => 12345678) - - # eval() is used to convert a string to regex - assert_match(eval("/" + key + "/"), admin.describe(@test_name)) - - command(:alter, @test_name, 'METHOD' => 'table_att_unset', 'NAME' => key) - assert_no_match(eval("/" + key + "/"), admin.describe(@test_name)) - end - - define_test "alter should be able to remove a list of table attributes" do - drop_test_table(@test_name) - - key_1 = "TestAttr1" - key_2 = "TestAttr2" - command(:create, @test_name, { NAME => 'i'}, METADATA => { key_1 => 1, key_2 => 2 }) - - # eval() is used to convert a string to regex - assert_match(eval("/" + key_1 + "/"), admin.describe(@test_name)) - assert_match(eval("/" + key_2 + "/"), admin.describe(@test_name)) - - command(:alter, @test_name, 'METHOD' => 'table_att_unset', 'NAME' => [ key_1, key_2 ]) - assert_no_match(eval("/" + key_1 + "/"), admin.describe(@test_name)) - assert_no_match(eval("/" + key_2 + "/"), admin.describe(@test_name)) - end - - define_test "alter should be able to remove a table configuration" do - drop_test_table(@test_name) - create_test_table(@test_name) - - key = "TestConf" - command(:alter, @test_name, CONFIGURATION => {key => 1}) - - # eval() is used to convert a string to regex - assert_match(eval("/" + key + "/"), admin.describe(@test_name)) - - command(:alter, @test_name, 'METHOD' => 'table_conf_unset', 'NAME' => key) - assert_no_match(eval("/" + key + "/"), admin.describe(@test_name)) - end - - define_test "alter should be able to remove a list of table configuration" do - drop_test_table(@test_name) - - key_1 = "TestConf1" - key_2 = "TestConf2" - command(:create, @test_name, { NAME => 'i'}, CONFIGURATION => { key_1 => 1, key_2 => 2 }) - - # eval() is used to convert a string to regex - assert_match(eval("/" + key_1 + "/"), admin.describe(@test_name)) - assert_match(eval("/" + key_2 + "/"), admin.describe(@test_name)) - - command(:alter, @test_name, 'METHOD' => 'table_conf_unset', 'NAME' => [ key_1, key_2 ]) - assert_no_match(eval("/" + key_1 + "/"), admin.describe(@test_name)) - assert_no_match(eval("/" + key_2 + "/"), admin.describe(@test_name)) - end - - define_test "get_table should get a real table" do - drop_test_table(@test_name) - create_test_table(@test_name) - - table = table(@test_name) - assert_not_equal(nil, table) - table.close - end - end # rubocop:enable Metrics/ClassLength end