From aa96377c8fc6e26e9051e03d20fc12c1a717f385 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Sun, 28 Jul 2024 13:51:18 +0800 Subject: [PATCH] HBASE-28719 Use ExtendedCell in WALEdit (#6108) Signed-off-by: Xin Sun (cherry picked from commit 836f2d9a5b3ae013ecd84efb457bf7ff20cdd997) --- ...ccessor.java => ClientInternalHelper.java} | 2 +- .../hbase/shaded/protobuf/ProtobufUtil.java | 6 +- .../apache/hadoop/hbase/mapreduce/Import.java | 8 +-- .../hadoop/hbase/mapreduce/PutCombiner.java | 6 +- .../hbase/mapreduce/PutSortReducer.java | 5 +- .../hbase/mapreduce/TestImportExport.java | 7 +-- .../hbase/mapreduce/TestWALRecordReader.java | 29 +++++---- .../hadoop/hbase/regionserver/HRegion.java | 34 ++++++----- .../hbase/regionserver/RSRpcServices.java | 14 ++--- .../hbase/regionserver/RegionScannerImpl.java | 4 +- .../hbase/regionserver/wal/FSWALEntry.java | 7 ++- .../ReplicationSourceWALReader.java | 6 +- .../VisibilityReplicationEndpoint.java | 18 +++--- .../org/apache/hadoop/hbase/wal/WALEdit.java | 59 +++++++++++++++---- .../hbase/wal/WALEditInternalHelper.java | 56 ++++++++++++++++++ .../apache/hadoop/hbase/wal/WALSplitUtil.java | 10 ++-- .../hbase/coprocessor/TestWALObserver.java | 4 +- .../hadoop/hbase/master/AbstractTestDLS.java | 3 +- .../hbase/regionserver/TestHRegion.java | 9 +-- .../TestRecoveredEditsReplayAndAbort.java | 4 +- .../regionserver/wal/AbstractTestFSWAL.java | 11 ++-- .../wal/AbstractTestWALReplay.java | 10 +++- .../wal/ProtobufLogTestHelper.java | 4 +- .../regionserver/wal/TestAsyncFSWAL.java | 4 +- .../wal/TestAsyncFSWALRollStuck.java | 10 ++-- .../regionserver/wal/TestFSWALEntry.java | 4 +- .../regionserver/wal/TestLogRollAbort.java | 4 +- .../wal/TestLogRollingNoCluster.java | 4 +- .../wal/TestWALActionsListener.java | 3 +- .../TestReplicationEmptyWALRecovery.java | 3 +- .../replication/TestReplicationEndpoint.java | 3 +- .../TestReplicationSmallTests.java | 4 +- .../TestReplicationWALEntryFilters.java | 3 +- .../master/TestRecoverStandbyProcedure.java | 4 +- .../regionserver/TestBasicWALEntryStream.java | 5 +- ...ClusterReplicationEndpointFilterEdits.java | 18 +++--- .../regionserver/TestReplicationSource.java | 12 ++-- .../TestReplicationSourceManager.java | 13 ++-- .../TestWALEntryStreamCompressionReset.java | 24 ++++---- .../regionserver/WALEntryStreamTestBase.java | 6 +- .../hbase/wal/CompressedWALTestBase.java | 6 +- .../hbase/wal/TestParsePartialWALFile.java | 6 +- .../hbase/wal/WALPerformanceEvaluation.java | 3 +- .../hadoop/hbase/thrift2/ThriftUtilities.java | 8 +-- 44 files changed, 305 insertions(+), 158 deletions(-) rename hbase-client/src/main/java/org/apache/hadoop/hbase/client/{PackagePrivateFieldAccessor.java => ClientInternalHelper.java} (98%) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALEditInternalHelper.java diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/PackagePrivateFieldAccessor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientInternalHelper.java similarity index 98% rename from hbase-client/src/main/java/org/apache/hadoop/hbase/client/PackagePrivateFieldAccessor.java rename to hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientInternalHelper.java index 08293ab83f81..cf3e344349aa 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/PackagePrivateFieldAccessor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientInternalHelper.java @@ -32,7 +32,7 @@ * TODO: A better solution is to separate the data structures used in client and server. */ @InterfaceAudience.Private -public class PackagePrivateFieldAccessor { +public class ClientInternalHelper { public static void setMvccReadPoint(Scan scan, long mvccReadPoint) { scan.setMvccReadPoint(mvccReadPoint); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java index d3672c5e841b..0dec61563494 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java @@ -76,6 +76,7 @@ import org.apache.hadoop.hbase.client.BalancerDecision; import org.apache.hadoop.hbase.client.BalancerRejection; import org.apache.hadoop.hbase.client.CheckAndMutate; +import org.apache.hadoop.hbase.client.ClientInternalHelper; import org.apache.hadoop.hbase.client.ClientUtil; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; @@ -89,7 +90,6 @@ import org.apache.hadoop.hbase.client.LogEntry; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.OnlineLogRecord; -import org.apache.hadoop.hbase.client.PackagePrivateFieldAccessor; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.client.RegionLoadStats; @@ -1082,7 +1082,7 @@ public static ClientProtos.Scan toScan(final Scan scan) throws IOException { if (scan.getCaching() > 0) { scanBuilder.setCaching(scan.getCaching()); } - long mvccReadPoint = PackagePrivateFieldAccessor.getMvccReadPoint(scan); + long mvccReadPoint = ClientInternalHelper.getMvccReadPoint(scan); if (mvccReadPoint > 0) { scanBuilder.setMvccReadPoint(mvccReadPoint); } @@ -1192,7 +1192,7 @@ public static Scan toScan(final ClientProtos.Scan proto) throws IOException { scan.setCaching(proto.getCaching()); } if (proto.hasMvccReadPoint()) { - PackagePrivateFieldAccessor.setMvccReadPoint(scan, proto.getMvccReadPoint()); + ClientInternalHelper.setMvccReadPoint(scan, proto.getMvccReadPoint()); } if (proto.hasReadType()) { scan.setReadType(toReadType(proto.getReadType())); diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java index 8a8b846959b6..ee09a7dc3972 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java @@ -47,12 +47,12 @@ import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.ZooKeeperConnectionException; import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ClientInternalHelper; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Durability; import org.apache.hadoop.hbase.client.Mutation; -import org.apache.hadoop.hbase.client.PackagePrivateFieldAccessor; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionLocator; import org.apache.hadoop.hbase.client.Result; @@ -205,7 +205,7 @@ public void map(ImmutableBytesWritable row, Result value, Context context) throw filter == null || !filter.filterRowKey( PrivateCellUtil.createFirstOnRow(row.get(), row.getOffset(), (short) row.getLength())) ) { - for (ExtendedCell kv : PackagePrivateFieldAccessor.getExtendedRawCells(value)) { + for (ExtendedCell kv : ClientInternalHelper.getExtendedRawCells(value)) { kv = filterKv(filter, kv); // skip if we filtered it out if (kv == null) { @@ -271,7 +271,7 @@ public void map(ImmutableBytesWritable row, Result value, Context context) throw filter == null || !filter.filterRowKey( PrivateCellUtil.createFirstOnRow(row.get(), row.getOffset(), (short) row.getLength())) ) { - for (ExtendedCell kv : PackagePrivateFieldAccessor.getExtendedRawCells(value)) { + for (ExtendedCell kv : ClientInternalHelper.getExtendedRawCells(value)) { kv = filterKv(filter, kv); // skip if we filtered it out if (kv == null) { @@ -336,7 +336,7 @@ private void writeResult(ImmutableBytesWritable key, Result result, Context cont protected void processKV(ImmutableBytesWritable key, Result result, Context context, Put put, Delete delete) throws IOException, InterruptedException { - for (ExtendedCell kv : PackagePrivateFieldAccessor.getExtendedRawCells(result)) { + for (ExtendedCell kv : ClientInternalHelper.getExtendedRawCells(result)) { kv = filterKv(filter, kv); // skip if we filter it out if (kv == null) { diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/PutCombiner.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/PutCombiner.java index cd25736bd6ee..4d79eca2a7e6 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/PutCombiner.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/PutCombiner.java @@ -22,7 +22,7 @@ import java.util.Map; import java.util.Map.Entry; import org.apache.hadoop.hbase.ExtendedCell; -import org.apache.hadoop.hbase.client.PackagePrivateFieldAccessor; +import org.apache.hadoop.hbase.client.ClientInternalHelper; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.mapreduce.Reducer; import org.apache.yetus.audience.InterfaceAudience; @@ -55,9 +55,9 @@ protected void reduce(K row, Iterable vals, Context context) cnt++; if (combinedPut == null) { combinedPut = p; - combinedFamilyMap = PackagePrivateFieldAccessor.getExtendedFamilyCellMap(combinedPut); + combinedFamilyMap = ClientInternalHelper.getExtendedFamilyCellMap(combinedPut); } else { - for (Entry> entry : PackagePrivateFieldAccessor + for (Entry> entry : ClientInternalHelper .getExtendedFamilyCellMap(p).entrySet()) { List existCells = combinedFamilyMap.get(entry.getKey()); if (existCells == null) { diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java index c8f32c205fb7..4cf7bcd9ff22 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java @@ -31,7 +31,7 @@ import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.TagType; import org.apache.hadoop.hbase.TagUtil; -import org.apache.hadoop.hbase.client.PackagePrivateFieldAccessor; +import org.apache.hadoop.hbase.client.ClientInternalHelper; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; @@ -100,8 +100,7 @@ protected void reduce(ImmutableBytesWritable row, Iterable puts, // just ignoring the bad one? throw new IOException("Invalid visibility expression found in mutation " + p, e); } - for (List cells : PackagePrivateFieldAccessor.getExtendedFamilyCellMap(p) - .values()) { + for (List cells : ClientInternalHelper.getExtendedFamilyCellMap(p).values()) { for (ExtendedCell cell : cells) { // Creating the KV which needs to be directly written to HFiles. Using the Facade // KVCreator for creation of kvs. diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java index 140f871e4386..63201e857398 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java @@ -53,6 +53,7 @@ import org.apache.hadoop.hbase.PrivateCellUtil; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Tag; +import org.apache.hadoop.hbase.client.ClientInternalHelper; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; @@ -60,7 +61,6 @@ import org.apache.hadoop.hbase.client.Durability; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Mutation; -import org.apache.hadoop.hbase.client.PackagePrivateFieldAccessor; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.Result; @@ -366,7 +366,7 @@ public void testWithDeletes() throws Throwable { s.setRaw(true); ResultScanner scanner = t.getScanner(s); Result r = scanner.next(); - ExtendedCell[] res = PackagePrivateFieldAccessor.getExtendedRawCells(r); + ExtendedCell[] res = ClientInternalHelper.getExtendedRawCells(r); assertTrue(PrivateCellUtil.isDeleteFamily(res[0])); assertEquals(now + 4, res[1].getTimestamp()); assertEquals(now + 3, res[2].getTimestamp()); @@ -934,8 +934,7 @@ public void testTagsWithEmptyCodec() throws Exception { int count = 0; Result result; while ((result = scanner.next()) != null) { - List cells = - Arrays.asList(PackagePrivateFieldAccessor.getExtendedRawCells(result)); + List cells = Arrays.asList(ClientInternalHelper.getExtendedRawCells(result)); assertEquals(2, cells.size()); ExtendedCell cell = cells.get(0); assertTrue(CellUtil.isDelete(cell)); diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALRecordReader.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALRecordReader.java index 795135cc6d19..3a457ee4d9c1 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALRecordReader.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALRecordReader.java @@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; +import org.apache.hadoop.hbase.wal.WALEditInternalHelper; import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hbase.wal.WALKey; import org.apache.hadoop.hbase.wal.WALKeyImpl; @@ -143,10 +144,12 @@ public void testPartialRead() throws Exception { // being millisecond based. long ts = EnvironmentEdgeManager.currentTime(); WALEdit edit = new WALEdit(); - edit.add(new KeyValue(rowName, family, Bytes.toBytes("1"), ts, value)); + WALEditInternalHelper.addExtendedCell(edit, + new KeyValue(rowName, family, Bytes.toBytes("1"), ts, value)); log.appendData(info, getWalKeyImpl(ts, scopes), edit); edit = new WALEdit(); - edit.add(new KeyValue(rowName, family, Bytes.toBytes("2"), ts + 1, value)); + WALEditInternalHelper.addExtendedCell(edit, + new KeyValue(rowName, family, Bytes.toBytes("2"), ts + 1, value)); log.appendData(info, getWalKeyImpl(ts + 1, scopes), edit); log.sync(); Threads.sleep(10); @@ -158,10 +161,12 @@ public void testPartialRead() throws Exception { long ts1 = EnvironmentEdgeManager.currentTime(); edit = new WALEdit(); - edit.add(new KeyValue(rowName, family, Bytes.toBytes("3"), ts1 + 1, value)); + WALEditInternalHelper.addExtendedCell(edit, + new KeyValue(rowName, family, Bytes.toBytes("3"), ts1 + 1, value)); log.appendData(info, getWalKeyImpl(ts1 + 1, scopes), edit); edit = new WALEdit(); - edit.add(new KeyValue(rowName, family, Bytes.toBytes("4"), ts1 + 2, value)); + WALEditInternalHelper.addExtendedCell(edit, + new KeyValue(rowName, family, Bytes.toBytes("4"), ts1 + 2, value)); log.appendData(info, getWalKeyImpl(ts1 + 2, scopes), edit); log.sync(); log.shutdown(); @@ -203,8 +208,8 @@ public void testWALRecordReader() throws Exception { WAL log = walfactory.getWAL(info); byte[] value = Bytes.toBytes("value"); WALEdit edit = new WALEdit(); - edit.add(new KeyValue(rowName, family, Bytes.toBytes("1"), EnvironmentEdgeManager.currentTime(), - value)); + WALEditInternalHelper.addExtendedCell(edit, new KeyValue(rowName, family, Bytes.toBytes("1"), + EnvironmentEdgeManager.currentTime(), value)); long txid = log.appendData(info, getWalKeyImpl(EnvironmentEdgeManager.currentTime(), scopes), edit); log.sync(txid); @@ -214,8 +219,8 @@ public void testWALRecordReader() throws Exception { log.rollWriter(); edit = new WALEdit(); - edit.add(new KeyValue(rowName, family, Bytes.toBytes("2"), EnvironmentEdgeManager.currentTime(), - value)); + WALEditInternalHelper.addExtendedCell(edit, new KeyValue(rowName, family, Bytes.toBytes("2"), + EnvironmentEdgeManager.currentTime(), value)); txid = log.appendData(info, getWalKeyImpl(EnvironmentEdgeManager.currentTime(), scopes), edit); log.sync(txid); log.shutdown(); @@ -261,8 +266,8 @@ public void testWALRecordReaderActiveArchiveTolerance() throws Exception { WAL log = walfactory.getWAL(info); byte[] value = Bytes.toBytes("value"); WALEdit edit = new WALEdit(); - edit.add(new KeyValue(rowName, family, Bytes.toBytes("1"), EnvironmentEdgeManager.currentTime(), - value)); + WALEditInternalHelper.addExtendedCell(edit, new KeyValue(rowName, family, Bytes.toBytes("1"), + EnvironmentEdgeManager.currentTime(), value)); long txid = log.appendData(info, getWalKeyImpl(EnvironmentEdgeManager.currentTime(), scopes), edit); log.sync(txid); @@ -270,8 +275,8 @@ public void testWALRecordReaderActiveArchiveTolerance() throws Exception { Thread.sleep(10); // make sure 2nd edit gets a later timestamp edit = new WALEdit(); - edit.add(new KeyValue(rowName, family, Bytes.toBytes("2"), EnvironmentEdgeManager.currentTime(), - value)); + WALEditInternalHelper.addExtendedCell(edit, new KeyValue(rowName, family, Bytes.toBytes("2"), + EnvironmentEdgeManager.currentTime(), value)); txid = log.appendData(info, getWalKeyImpl(EnvironmentEdgeManager.currentTime(), scopes), edit); log.sync(txid); log.shutdown(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 68f5356f5549..b74935ffafff 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -104,6 +104,7 @@ import org.apache.hadoop.hbase.client.Append; import org.apache.hadoop.hbase.client.CheckAndMutate; import org.apache.hadoop.hbase.client.CheckAndMutateResult; +import org.apache.hadoop.hbase.client.ClientInternalHelper; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.CompactionState; import org.apache.hadoop.hbase.client.Delete; @@ -112,7 +113,6 @@ import org.apache.hadoop.hbase.client.Increment; import org.apache.hadoop.hbase.client.IsolationLevel; import org.apache.hadoop.hbase.client.Mutation; -import org.apache.hadoop.hbase.client.PackagePrivateFieldAccessor; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionReplicaUtil; @@ -181,6 +181,7 @@ import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; +import org.apache.hadoop.hbase.wal.WALEditInternalHelper; import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hbase.wal.WALKey; import org.apache.hadoop.hbase.wal.WALKeyImpl; @@ -3513,7 +3514,7 @@ protected void checkAndPrepareMutation(int index, long timestamp) throws IOExcep // store the family map reference to allow for mutations // we know that in mutation, only ExtendedCells are allow so here we do a fake cast, to // simplify later logic - familyCellMaps[index] = PackagePrivateFieldAccessor.getExtendedFamilyCellMap(mutation); + familyCellMaps[index] = ClientInternalHelper.getExtendedFamilyCellMap(mutation); } // store durability for the batch (highest durability of all operations in the batch) @@ -3709,7 +3710,9 @@ public boolean visit(int index) throws IOException { // Add WAL edits from CPs. WALEdit fromCP = walEditsFromCoprocessors[index]; - List cellsFromCP = fromCP == null ? Collections.emptyList() : fromCP.getCells(); + List cellsFromCP = fromCP == null + ? Collections.emptyList() + : WALEditInternalHelper.getExtendedCells(fromCP); addNonSkipWALMutationsToWALEdit(miniBatchOp, walEdit, cellsFromCP, familyCellMaps[index]); return true; } @@ -3719,14 +3722,14 @@ public boolean visit(int index) throws IOException { protected void addNonSkipWALMutationsToWALEdit( final MiniBatchOperationInProgress miniBatchOp, WALEdit walEdit, - List cellsFromCP, Map> familyCellMap) { + List cellsFromCP, Map> familyCellMap) { doAddCellsToWALEdit(walEdit, cellsFromCP, familyCellMap); } - protected static void doAddCellsToWALEdit(WALEdit walEdit, List cellsFromCP, + protected static void doAddCellsToWALEdit(WALEdit walEdit, List cellsFromCP, Map> familyCellMap) { - walEdit.add(cellsFromCP); - walEdit.add((Map) familyCellMap); + WALEditInternalHelper.addExtendedCell(walEdit, cellsFromCP); + WALEditInternalHelper.addMap(walEdit, familyCellMap); } protected abstract void cacheSkipWALMutationForRegionReplication( @@ -4064,7 +4067,7 @@ private Map> reckonDeltas(Mutation mutation, assert mutation instanceof Increment || mutation instanceof Append; Map> ret = new TreeMap<>(Bytes.BYTES_COMPARATOR); // Process a Store/family at a time. - for (Map.Entry> entry : PackagePrivateFieldAccessor + for (Map.Entry> entry : ClientInternalHelper .getExtendedFamilyCellMap(mutation).entrySet()) { final byte[] columnFamilyName = entry.getKey(); List deltas = (List) entry.getValue(); @@ -4260,7 +4263,7 @@ protected void cacheSkipWALMutationForRegionReplication( this.createWALEditForReplicateSkipWAL(miniBatchOp, nonceKeyAndWALEdits); miniBatchOp.setWalEditForReplicateIfExistsSkipWAL(walEditForReplicateIfExistsSkipWAL); } - walEditForReplicateIfExistsSkipWAL.add((Map) familyCellMap); + WALEditInternalHelper.addMap(walEditForReplicateIfExistsSkipWAL, familyCellMap); } @@ -4279,8 +4282,7 @@ private WALEdit createWALEditForReplicateSkipWAL( @Override protected void addNonSkipWALMutationsToWALEdit( final MiniBatchOperationInProgress miniBatchOp, WALEdit walEdit, - List cellsFromCP, Map> familyCellMap) { - + List cellsFromCP, Map> familyCellMap) { super.addNonSkipWALMutationsToWALEdit(miniBatchOp, walEdit, cellsFromCP, familyCellMap); WALEdit walEditForReplicateIfExistsSkipWAL = miniBatchOp.getWalEditForReplicateIfExistsSkipWAL(); @@ -4524,7 +4526,7 @@ private void checkAndMergeCPMutations(final MiniBatchOperationInProgress> cpFamilyMap = - PackagePrivateFieldAccessor.getExtendedFamilyCellMap(cpMutation); + ClientInternalHelper.getExtendedFamilyCellMap(cpMutation); region.rewriteCellTags(cpFamilyMap, mutation); // will get added to the memStore later mergeFamilyMaps(familyCellMaps[i], cpFamilyMap); @@ -5096,16 +5098,16 @@ private CheckAndMutateResult checkAndMutateInternal(CheckAndMutate checkAndMutat byte[] byteTs = Bytes.toBytes(ts); if (mutation != null) { if (mutation instanceof Put) { - updateCellTimestamps( - PackagePrivateFieldAccessor.getExtendedFamilyCellMap(mutation).values(), byteTs); + updateCellTimestamps(ClientInternalHelper.getExtendedFamilyCellMap(mutation).values(), + byteTs); } // And else 'delete' is not needed since it already does a second get, and sets the // timestamp from get (see prepareDeleteTimestamps). } else { for (Mutation m : rowMutations.getMutations()) { if (m instanceof Put) { - updateCellTimestamps( - PackagePrivateFieldAccessor.getExtendedFamilyCellMap(m).values(), byteTs); + updateCellTimestamps(ClientInternalHelper.getExtendedFamilyCellMap(m).values(), + byteTs); } } // And else 'delete' is not needed since it already does a second get, and sets the diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java index 5e16b08369b1..ce9cab6bf3b2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java @@ -67,13 +67,13 @@ import org.apache.hadoop.hbase.client.Append; import org.apache.hadoop.hbase.client.CheckAndMutate; import org.apache.hadoop.hbase.client.CheckAndMutateResult; +import org.apache.hadoop.hbase.client.ClientInternalHelper; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Durability; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Increment; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.OperationWithAttributes; -import org.apache.hadoop.hbase.client.PackagePrivateFieldAccessor; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionReplicaUtil; @@ -2078,7 +2078,7 @@ private ExtendedCellScanner getAndReset(RpcController controller) { public ReplicateWALEntryResponse replay(final RpcController controller, final ReplicateWALEntryRequest request) throws ServiceException { long before = EnvironmentEdgeManager.currentTime(); - CellScanner cells = getAndReset(controller); + ExtendedCellScanner cells = getAndReset(controller); try { checkOpen(); List entries = request.getEntryList(); @@ -2500,8 +2500,8 @@ public GetResponse get(final RpcController controller, final GetRequest request) && VersionInfoUtil.hasMinimumVersion(context.getClientVersionInfo(), 1, 3) ) { pbr = ProtobufUtil.toResultNoData(r); - ((HBaseRpcController) controller).setCellScanner(PrivateCellUtil - .createExtendedCellScanner(PackagePrivateFieldAccessor.getExtendedRawCells(r))); + ((HBaseRpcController) controller).setCellScanner( + PrivateCellUtil.createExtendedCellScanner(ClientInternalHelper.getExtendedRawCells(r))); addSize(context, r); } else { pbr = ProtobufUtil.toResult(r); @@ -3426,10 +3426,8 @@ private void scan(HBaseRpcController controller, ScanRequest request, RegionScan int lastIdx = results.size() - 1; Result r = results.get(lastIdx); if (r.mayHaveMoreCellsInRow()) { - results.set(lastIdx, - PackagePrivateFieldAccessor.createResult( - PackagePrivateFieldAccessor.getExtendedRawCells(r), r.getExists(), r.isStale(), - false)); + results.set(lastIdx, ClientInternalHelper.createResult( + ClientInternalHelper.getExtendedRawCells(r), r.getExists(), r.isStale(), false)); } } boolean sizeLimitReached = scannerContext.checkSizeLimit(LimitScope.BETWEEN_ROWS); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScannerImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScannerImpl.java index a5fc2947bee0..81b5f6a6d70c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScannerImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScannerImpl.java @@ -34,8 +34,8 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.PrivateCellUtil; import org.apache.hadoop.hbase.UnknownScannerException; +import org.apache.hadoop.hbase.client.ClientInternalHelper; import org.apache.hadoop.hbase.client.IsolationLevel; -import org.apache.hadoop.hbase.client.PackagePrivateFieldAccessor; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.filter.FilterWrapper; @@ -128,7 +128,7 @@ private static boolean hasNonce(HRegion region, long nonce) { // synchronize on scannerReadPoints so that nobody calculates // getSmallestReadPoint, before scannerReadPoints is updated. IsolationLevel isolationLevel = scan.getIsolationLevel(); - long mvccReadPoint = PackagePrivateFieldAccessor.getMvccReadPoint(scan); + long mvccReadPoint = ClientInternalHelper.getMvccReadPoint(scan); this.scannerReadPoints = region.scannerReadPoints; this.rsServices = region.getRegionServerServices(); region.smallestReadPointCalcLock.lock(ReadPointCalculationLock.LockType.RECORDING_LOCK); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSWALEntry.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSWALEntry.java index 502e04e05b06..8fa83ba08bf9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSWALEntry.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSWALEntry.java @@ -24,6 +24,7 @@ import java.util.TreeSet; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.ExtendedCell; import org.apache.hadoop.hbase.PrivateCellUtil; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.ipc.ServerCall; @@ -31,6 +32,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.wal.WAL.Entry; import org.apache.hadoop.hbase.wal.WALEdit; +import org.apache.hadoop.hbase.wal.WALEditInternalHelper; import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.yetus.audience.InterfaceAudience; @@ -80,7 +82,8 @@ class FSWALEntry extends Entry { if (inMemstore) { // construct familyNames here to reduce the work of log sinker. Set families = edit.getFamilies(); - this.familyNames = families != null ? families : collectFamilies(edit.getCells()); + this.familyNames = + families != null ? families : collectFamilies(WALEditInternalHelper.getExtendedCells(edit)); } else { this.familyNames = Collections.emptySet(); } @@ -90,7 +93,7 @@ class FSWALEntry extends Entry { } } - static Set collectFamilies(List cells) { + static Set collectFamilies(List cells) { if (CollectionUtils.isEmpty(cells)) { return Collections.emptySet(); } else { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java index bd5b7736f3b9..fe983c9f3ae6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java @@ -28,12 +28,14 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.ExtendedCell; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.replication.WALEntryFilter; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.wal.WAL.Entry; import org.apache.hadoop.hbase.wal.WALEdit; +import org.apache.hadoop.hbase.wal.WALEditInternalHelper; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; import org.slf4j.Logger; @@ -413,10 +415,10 @@ private void updateReplicationMarkerEdit(Entry entry, long offset) { // Create a new KeyValue KeyValue kv = new KeyValue(CellUtil.cloneRow(cell), CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell), cell.getTimestamp(), descriptor.toByteArray()); - ArrayList newCells = new ArrayList<>(); + ArrayList newCells = new ArrayList<>(); newCells.add(kv); // Update edit with new cell. - edit.setCells(newCells); + WALEditInternalHelper.setExtendedCells(edit, newCells); } /** Returns whether the reader thread is running */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityReplicationEndpoint.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityReplicationEndpoint.java index 1b91ed718f61..b97a08c01c38 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityReplicationEndpoint.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityReplicationEndpoint.java @@ -24,7 +24,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import org.apache.hadoop.hbase.ArrayBackedTag; -import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.ExtendedCell; import org.apache.hadoop.hbase.PrivateCellUtil; import org.apache.hadoop.hbase.Tag; @@ -34,6 +33,7 @@ import org.apache.hadoop.hbase.replication.WALEntryFilter; import org.apache.hadoop.hbase.wal.WAL.Entry; import org.apache.hadoop.hbase.wal.WALEdit; +import org.apache.hadoop.hbase.wal.WALEditInternalHelper; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -75,10 +75,8 @@ public boolean replicate(ReplicateContext replicateContext) { List newEntries = new ArrayList<>(entries.size()); for (Entry entry : entries) { WALEdit newEdit = new WALEdit(); - ArrayList cells = entry.getEdit().getCells(); - for (Cell c : cells) { - assert c instanceof ExtendedCell; - ExtendedCell cell = (ExtendedCell) c; + List cells = WALEditInternalHelper.getExtendedCells(entry.getEdit()); + for (ExtendedCell cell : cells) { if (cell.getTagsLength() > 0) { visTags.clear(); nonVisTags.clear(); @@ -99,17 +97,17 @@ public boolean replicate(ReplicateContext replicateContext) { + "string type for the cell " + cell + ".", ioe); // just return the old entries as it is without applying the string type change - newEdit.add(cell); + WALEditInternalHelper.addExtendedCell(newEdit, cell); continue; } // Recreate the cell with the new tags and the existing tags - Cell newCell = PrivateCellUtil.createCell(cell, nonVisTags); - newEdit.add(newCell); + ExtendedCell newCell = PrivateCellUtil.createCell(cell, nonVisTags); + WALEditInternalHelper.addExtendedCell(newEdit, newCell); } else { - newEdit.add(cell); + WALEditInternalHelper.addExtendedCell(newEdit, cell); } } else { - newEdit.add(cell); + WALEditInternalHelper.addExtendedCell(newEdit, cell); } } newEntries.add(new Entry((entry.getKey()), newEdit)); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALEdit.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALEdit.java index 01dbe06682bb..512e21803b31 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALEdit.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALEdit.java @@ -25,6 +25,7 @@ import java.util.TreeSet; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.ExtendedCell; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.PrivateCellUtil; @@ -145,7 +146,7 @@ public class WALEdit implements HeapSize { private final transient boolean replay; - private ArrayList cells; + private ArrayList cells; /** * All the Cell families in cells. Updated by {@link #add(Cell)} and @@ -249,21 +250,29 @@ public boolean isReplay() { } public WALEdit add(Cell cell, byte[] family) { + return add(PrivateCellUtil.ensureExtendedCell(cell), family); + } + + WALEdit add(ExtendedCell cell, byte[] family) { getOrCreateFamilies().add(family); return addCell(cell); } public WALEdit add(Cell cell) { + return add(PrivateCellUtil.ensureExtendedCell(cell)); + } + + WALEdit add(ExtendedCell cell) { // We clone Family each time we add a Cell. Expensive but safe. For CPU savings, use // add(Map) or add(Cell, family). return add(cell, CellUtil.cloneFamily(cell)); } - public WALEdit add(List cells) { + WALEdit add(List cells) { if (cells == null || cells.isEmpty()) { return this; } - for (Cell cell : cells) { + for (ExtendedCell cell : cells) { add(cell); } return this; @@ -278,16 +287,29 @@ public int size() { } public ArrayList getCells() { + return (ArrayList) cells; + } + + List getExtendedCells() { return cells; } + /** + * This is just for keeping compatibility for CPs, in HBase you should call the below + * {@link #setExtendedCells(ArrayList)} directly to avoid casting. + */ + void setCells(ArrayList cells) { + this.cells = new ArrayList<>((ArrayList) cells); + this.families = null; + } + /** * This is not thread safe. This will change the WALEdit and shouldn't be used unless you are sure * that nothing else depends on the contents being immutable. * @param cells the list of cells that this WALEdit now contains. */ // Used by replay. - public void setCells(ArrayList cells) { + void setExtendedCells(ArrayList cells) { this.cells = cells; this.families = null; } @@ -458,14 +480,31 @@ public static WALProtos.BulkLoadDescriptor getBulkLoadDescriptor(Cell cell) thro } /** - * Append the given map of family->edits to a WALEdit data structure. This does not write to the - * WAL itself. Note that as an optimization, we will stamp the Set of column families into the - * WALEdit to save on our having to calculate column families subsequently down in the actual WAL - * writing. - * @param familyMap map of family->edits + * This is just for keeping compatibility for CPs, in HBase you should call the below + * {@link #addMap(Map)} directly to avoid casting. */ public void add(Map> familyMap) { for (Map.Entry> e : familyMap.entrySet()) { + // 'foreach' loop NOT used. See HBASE-12023 "...creates too many iterator objects." + int listSize = e.getValue().size(); + // Add all Cells first and then at end, add the family rather than call {@link #add(Cell)} + // and have it clone family each time. Optimization! + for (int i = 0; i < listSize; i++) { + addCell(PrivateCellUtil.ensureExtendedCell(e.getValue().get(i))); + } + addFamily(e.getKey()); + } + } + + /** + * Append the given map of family-> edits to a WALEdit data structure. This does not write to + * the WAL itself. Note that as an optimization, we will stamp the Set of column families into the + * WALEdit to save on our having to calculate column families subsequently down in the actual WAL + * writing. + * @param familyMap map of family -> edits + */ + void addMap(Map> familyMap) { + for (Map.Entry> e : familyMap.entrySet()) { // 'foreach' loop NOT used. See HBASE-12023 "...creates too many iterator objects." int listSize = e.getValue().size(); // Add all Cells first and then at end, add the family rather than call {@link #add(Cell)} @@ -481,7 +520,7 @@ private void addFamily(byte[] family) { getOrCreateFamilies().add(family); } - private WALEdit addCell(Cell cell) { + private WALEdit addCell(ExtendedCell cell) { this.cells.add(cell); return this; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALEditInternalHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALEditInternalHelper.java new file mode 100644 index 000000000000..0aba676f6dcf --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALEditInternalHelper.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.wal; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.hbase.ExtendedCell; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * A helper class so we can call some package private methods of {@link WALEdit} from other + * packages. Since {@link WALEdit} has been exposed to coprocessor and replication implementations, + * we do not want to make all the methods in it public. + */ +@InterfaceAudience.Private +public final class WALEditInternalHelper { + + private WALEditInternalHelper() { + } + + public static WALEdit addExtendedCell(WALEdit edit, ExtendedCell cell) { + return edit.add(cell); + } + + public static void addExtendedCell(WALEdit edit, List cells) { + edit.add(cells); + } + + public static void addMap(WALEdit edit, Map> familyMap) { + edit.addMap(familyMap); + } + + public static void setExtendedCells(WALEdit edit, ArrayList cells) { + edit.setExtendedCells(cells); + } + + public static List getExtendedCells(WALEdit edit) { + return edit.getExtendedCells(); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitUtil.java index cbfde9c7e172..fab9936165d1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitUtil.java @@ -36,8 +36,9 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.ExtendedCell; +import org.apache.hadoop.hbase.ExtendedCellScanner; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Delete; @@ -477,7 +478,8 @@ public ClientProtos.MutationProto.MutationType getType() { */ @Deprecated public static List getMutationsFromWALEntry(AdminProtos.WALEntry entry, - CellScanner cells, Pair logEntry, Durability durability) throws IOException { + ExtendedCellScanner cells, Pair logEntry, Durability durability) + throws IOException { if (entry == null) { // return an empty array return Collections.emptyList(); @@ -501,9 +503,9 @@ public static List getMutationsFromWALEntry(AdminProtos.WALEntry if (!cells.advance()) { throw new ArrayIndexOutOfBoundsException("Expected=" + count + ", index=" + i); } - Cell cell = cells.current(); + ExtendedCell cell = cells.current(); if (val != null) { - val.add(cell); + WALEditInternalHelper.addExtendedCell(val, cell); } boolean isNewRowOrType = diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java index 105c57b55ea0..989110e41d97 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java @@ -61,6 +61,7 @@ import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; +import org.apache.hadoop.hbase.wal.WALEditInternalHelper; import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.wal.WALSplitter; @@ -444,7 +445,8 @@ private void addWALEdits(final TableName tableName, final RegionInfo hri, final byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j)); byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j)); WALEdit edit = new WALEdit(); - edit.add(new KeyValue(rowName, family, qualifierBytes, ee.currentTime(), columnBytes)); + WALEditInternalHelper.addExtendedCell(edit, + new KeyValue(rowName, family, qualifierBytes, ee.currentTime(), columnBytes)); // uses WALKeyImpl instead of HLogKey on purpose. will only work for tests where we don't care // about legacy coprocessors txid = wal.appendData(hri, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/AbstractTestDLS.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/AbstractTestDLS.java index 00f6f2005806..419db220e17f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/AbstractTestDLS.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/AbstractTestDLS.java @@ -66,6 +66,7 @@ import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; +import org.apache.hadoop.hbase.wal.WALEditInternalHelper; import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.zookeeper.ZKUtil; import org.junit.After; @@ -434,7 +435,7 @@ public void makeWAL(HRegionServer hrs, List regions, int numEdits, i row = Arrays.copyOfRange(row, 3, 8); // use last 5 bytes because // HBaseTestingUtility.createMultiRegions use 5 bytes key byte[] qualifier = Bytes.toBytes("c" + Integer.toString(i)); - e.add( + WALEditInternalHelper.addExtendedCell(e, new KeyValue(row, COLUMN_FAMILY, qualifier, EnvironmentEdgeManager.currentTime(), value)); log.appendData(curRegionInfo, new WALKeyImpl(curRegionInfo.getEncodedNameAsBytes(), tableName, EnvironmentEdgeManager.currentTime(), mvcc), e); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java index d9856b40a831..99c811e720c0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java @@ -171,6 +171,7 @@ import org.apache.hadoop.hbase.wal.NettyAsyncFSWALConfigHelper; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; +import org.apache.hadoop.hbase.wal.WALEditInternalHelper; import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.wal.WALProvider; @@ -699,7 +700,7 @@ public void testArchiveRecoveredEditsReplay() throws Exception { long time = System.nanoTime(); WALEdit edit = new WALEdit(); - edit.add( + WALEditInternalHelper.addExtendedCell(edit, new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i))); writer.append(new WAL.Entry( new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); @@ -753,7 +754,7 @@ public void testSkipRecoveredEditsReplay() throws Exception { long time = System.nanoTime(); WALEdit edit = new WALEdit(); - edit.add( + WALEditInternalHelper.addExtendedCell(edit, new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i))); writer.append(new WAL.Entry( new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); @@ -804,7 +805,7 @@ public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception { long time = System.nanoTime(); WALEdit edit = new WALEdit(); - edit.add( + WALEditInternalHelper.addExtendedCell(edit, new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i))); writer.append(new WAL.Entry( new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); @@ -899,7 +900,7 @@ public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception { .setRegionName(ByteString.copyFrom(region.getRegionInfo().getRegionName())).build()); } else { edit = new WALEdit(); - edit.add( + WALEditInternalHelper.addExtendedCell(edit, new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i))); } writer.append(new WAL.Entry( diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRecoveredEditsReplayAndAbort.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRecoveredEditsReplayAndAbort.java index c54307c66fc5..6b372fa99350 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRecoveredEditsReplayAndAbort.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRecoveredEditsReplayAndAbort.java @@ -40,6 +40,7 @@ import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; +import org.apache.hadoop.hbase.wal.WALEditInternalHelper; import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.wal.WALProvider; @@ -147,7 +148,8 @@ public void test() throws Exception { // 200KB kv byte[] value = new byte[200 * 1024]; Bytes.random(value); - edit.add(new KeyValue(row, fam1, Bytes.toBytes(j), time, KeyValue.Type.Put, value)); + WALEditInternalHelper.addExtendedCell(edit, + new KeyValue(row, fam1, Bytes.toBytes(j), time, KeyValue.Type.Put, value)); writer.append(new WAL.Entry( new WALKeyImpl(regionName, tableName, j, time, HConstants.DEFAULT_CLUSTER_ID), edit)); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java index fd80f6cceae4..e8a364cd54ca 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java @@ -50,8 +50,8 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.Coprocessor; +import org.apache.hadoop.hbase.ExtendedCellScanner; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.HConstants; @@ -87,6 +87,7 @@ import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; +import org.apache.hadoop.hbase.wal.WALEditInternalHelper; import org.apache.hadoop.hbase.wal.WALKey; import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.junit.AfterClass; @@ -183,7 +184,7 @@ protected void addEdits(WAL log, RegionInfo hri, TableDescriptor htd, int times, for (int i = 0; i < times; i++) { long timestamp = EnvironmentEdgeManager.currentTime(); WALEdit cols = new WALEdit(); - cols.add(new KeyValue(row, row, row, timestamp, row)); + WALEditInternalHelper.addExtendedCell(cols, new KeyValue(row, row, row, timestamp, row)); WALKeyImpl key = new WALKeyImpl(hri.getEncodedNameAsBytes(), htd.getTableName(), SequenceId.NO_SEQUENCE_ID, timestamp, WALKey.EMPTY_UUIDS, HConstants.NO_NONCE, HConstants.NO_NONCE, mvcc, scopes); @@ -459,9 +460,9 @@ public void run() { // Construct a WALEdit and add it a few times to the WAL. WALEdit edits = new WALEdit(); for (Put p : puts) { - CellScanner cs = p.cellScanner(); + ExtendedCellScanner cs = p.cellScanner(); while (cs.advance()) { - edits.add(cs.current()); + WALEditInternalHelper.addExtendedCell(edits, cs.current()); } } // Add any old cluster id. @@ -517,7 +518,7 @@ public void testWriteEntryCanBeNull() throws IOException { long timestamp = EnvironmentEdgeManager.currentTime(); byte[] row = Bytes.toBytes("row"); WALEdit cols = new WALEdit(); - cols.add(new KeyValue(row, row, row, timestamp, row)); + WALEditInternalHelper.addExtendedCell(cols, new KeyValue(row, row, row, timestamp, row)); WALKeyImpl key = new WALKeyImpl(ri.getEncodedNameAsBytes(), td.getTableName(), SequenceId.NO_SEQUENCE_ID, timestamp, WALKey.EMPTY_UUIDS, HConstants.NO_NONCE, HConstants.NO_NONCE, mvcc, scopes); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java index 947c14e716f3..18b560519bb5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java @@ -99,6 +99,7 @@ import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; +import org.apache.hadoop.hbase.wal.WALEditInternalHelper; import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.wal.WALSplitUtil; @@ -757,14 +758,16 @@ public void testReplayEditsWrittenIntoWAL() throws Exception { // Add an edit to another family, should be skipped. WALEdit edit = new WALEdit(); long now = ee.currentTime(); - edit.add(new KeyValue(rowName, Bytes.toBytes("another family"), rowName, now, rowName)); + WALEditInternalHelper.addExtendedCell(edit, + new KeyValue(rowName, Bytes.toBytes("another family"), rowName, now, rowName)); wal.appendData(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), tableName, now, mvcc, scopes), edit); // Delete the c family to verify deletes make it over. edit = new WALEdit(); now = ee.currentTime(); - edit.add(new KeyValue(rowName, Bytes.toBytes("c"), null, now, KeyValue.Type.DeleteFamily)); + WALEditInternalHelper.addExtendedCell(edit, + new KeyValue(rowName, Bytes.toBytes("c"), null, now, KeyValue.Type.DeleteFamily)); wal.appendData(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), tableName, now, mvcc, scopes), edit); @@ -1103,7 +1106,8 @@ private WALEdit createWALEdit(final byte[] rowName, final byte[] family, Environ byte[] qualifierBytes = Bytes.toBytes(Integer.toString(index)); byte[] columnBytes = Bytes.toBytes(Bytes.toString(family) + ":" + Integer.toString(index)); WALEdit edit = new WALEdit(); - edit.add(new KeyValue(rowName, family, qualifierBytes, ee.currentTime(), columnBytes)); + WALEditInternalHelper.addExtendedCell(edit, + new KeyValue(rowName, family, qualifierBytes, ee.currentTime(), columnBytes)); return edit; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogTestHelper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogTestHelper.java index becb0f1794dd..a70f6242432a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogTestHelper.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogTestHelper.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; +import org.apache.hadoop.hbase.wal.WALEditInternalHelper; import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.wal.WALProvider; @@ -64,7 +65,8 @@ private static WAL.Entry generateEdit(int i, RegionInfo hri, TableName tableName WALEdit edit = new WALEdit(); int prefix = i; IntStream.range(0, columnCount).mapToObj(j -> toValue(prefix, j)) - .map(value -> new KeyValue(row, row, row, timestamp, value)).forEachOrdered(edit::add); + .map(value -> new KeyValue(row, row, row, timestamp, value)) + .forEachOrdered(c -> WALEditInternalHelper.addExtendedCell(edit, c)); return new WAL.Entry(key, edit); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncFSWAL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncFSWAL.java index dc075ff8f966..b4e5d3402ae2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncFSWAL.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncFSWAL.java @@ -54,6 +54,7 @@ import org.apache.hadoop.hbase.util.FutureUtils; import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.wal.WALEdit; +import org.apache.hadoop.hbase.wal.WALEditInternalHelper; import org.apache.hadoop.hbase.wal.WALKey; import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.wal.WALProvider.AsyncWriter; @@ -202,7 +203,8 @@ public void append(Entry entry) { public void run() { byte[] row = Bytes.toBytes("row" + index); WALEdit cols = new WALEdit(); - cols.add(new KeyValue(row, row, row, timestamp + index, row)); + WALEditInternalHelper.addExtendedCell(cols, + new KeyValue(row, row, row, timestamp + index, row)); WALKeyImpl key = new WALKeyImpl(ri.getEncodedNameAsBytes(), td.getTableName(), SequenceId.NO_SEQUENCE_ID, timestamp, WALKey.EMPTY_UUIDS, HConstants.NO_NONCE, HConstants.NO_NONCE, mvcc, scopes); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncFSWALRollStuck.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncFSWALRollStuck.java index 510814ed1279..931362832ed0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncFSWALRollStuck.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncFSWALRollStuck.java @@ -31,8 +31,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell.Type; -import org.apache.hadoop.hbase.CellBuilderFactory; import org.apache.hadoop.hbase.CellBuilderType; +import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.TableName; @@ -47,6 +47,7 @@ import org.apache.hadoop.hbase.wal.AsyncFSWALProvider; import org.apache.hadoop.hbase.wal.AsyncFSWALProvider.AsyncWriter; import org.apache.hadoop.hbase.wal.WALEdit; +import org.apache.hadoop.hbase.wal.WALEditInternalHelper; import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -181,9 +182,10 @@ public static void tearDown() throws Exception { public void testRoll() throws Exception { byte[] row = Bytes.toBytes("family"); WALEdit edit = new WALEdit(); - edit.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setFamily(row) - .setQualifier(row).setRow(row).setValue(row) - .setTimestamp(EnvironmentEdgeManager.currentTime()).setType(Type.Put).build()); + WALEditInternalHelper.addExtendedCell(edit, + ExtendedCellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setFamily(row) + .setQualifier(row).setRow(row).setValue(row) + .setTimestamp(EnvironmentEdgeManager.currentTime()).setType(Type.Put).build()); WALKeyImpl key1 = new WALKeyImpl(RI.getEncodedNameAsBytes(), TN, EnvironmentEdgeManager.currentTime(), MVCC); WAL.appendData(RI, key1, edit); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSWALEntry.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSWALEntry.java index 4d0687829ac0..1a0ecdfc55d7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSWALEntry.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSWALEntry.java @@ -21,8 +21,8 @@ import java.util.ArrayList; import java.util.List; -import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellBuilderType; +import org.apache.hadoop.hbase.ExtendedCell; import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HConstants; @@ -48,7 +48,7 @@ public void testCollectFamilies() { byte[] family1 = Bytes.toBytes("family1"); byte[] family2 = Bytes.toBytes("family2"); - List cells = new ArrayList<>(); + List cells = new ArrayList<>(); assertEquals(0, FSWALEntry.collectFamilies(cells).size()); cells.add(ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(family0) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java index 90f595003cb1..3c3dbe1ead9e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java @@ -55,6 +55,7 @@ import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; +import org.apache.hadoop.hbase.wal.WALEditInternalHelper; import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.wal.WALProvider; @@ -225,7 +226,8 @@ public void testLogRollAfterSplitStart() throws IOException { int total = 20; for (int i = 0; i < total; i++) { WALEdit kvs = new WALEdit(); - kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName())); + WALEditInternalHelper.addExtendedCell(kvs, + new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName())); NavigableMap scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR); scopes.put(Bytes.toBytes("column"), 0); log.appendData(regionInfo, new WALKeyImpl(regionInfo.getEncodedNameAsBytes(), tableName, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollingNoCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollingNoCluster.java index 606ee55c3159..dd4fe77c8a38 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollingNoCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollingNoCluster.java @@ -45,6 +45,7 @@ import org.apache.hadoop.hbase.wal.FSHLogProvider; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; +import org.apache.hadoop.hbase.wal.WALEditInternalHelper; import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.junit.ClassRule; @@ -167,7 +168,8 @@ public void run() { } WALEdit edit = new WALEdit(); byte[] bytes = Bytes.toBytes(i); - edit.add(new KeyValue(bytes, bytes, bytes, now, EMPTY_1K_ARRAY)); + WALEditInternalHelper.addExtendedCell(edit, + new KeyValue(bytes, bytes, bytes, now, EMPTY_1K_ARRAY)); RegionInfo hri = RegionInfoBuilder.FIRST_META_REGIONINFO; NavigableMap scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR); for (byte[] fam : this.metaTableDescriptor.getColumnFamilyNames()) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java index 0ad2e8d0522c..61dd8def4ea6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; +import org.apache.hadoop.hbase.wal.WALEditInternalHelper; import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.junit.After; @@ -107,7 +108,7 @@ public void testActionListener() throws Exception { byte[] b = Bytes.toBytes(i + ""); KeyValue kv = new KeyValue(b, b, b); WALEdit edit = new WALEdit(); - edit.add(kv); + WALEditInternalHelper.addExtendedCell(edit, kv); NavigableMap scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR); scopes.put(b, 0); long txid = wal.appendData(hri, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationEmptyWALRecovery.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationEmptyWALRecovery.java index 67546febab72..38fc6599dad0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationEmptyWALRecovery.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationEmptyWALRecovery.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; +import org.apache.hadoop.hbase.wal.WALEditInternalHelper; import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.junit.Assert; import org.junit.Before; @@ -361,7 +362,7 @@ private void appendEntriesToWal(int numEntries, WAL wal) throws IOException { byte[] b = Bytes.toBytes(Integer.toString(i)); KeyValue kv = new KeyValue(b, famName, b); WALEdit edit = new WALEdit(); - edit.add(kv); + WALEditInternalHelper.addExtendedCell(edit, kv); txId = wal.appendData(info, getWalKeyImpl(), edit); } wal.sync(txId); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationEndpoint.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationEndpoint.java index 9bc632e223be..057a9f3567f5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationEndpoint.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationEndpoint.java @@ -62,6 +62,7 @@ import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.wal.WAL.Entry; import org.apache.hadoop.hbase.wal.WALEdit; +import org.apache.hadoop.hbase.wal.WALEditInternalHelper; import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.zookeeper.ZKConfig; import org.apache.hadoop.metrics2.lib.DynamicMetricsRegistry; @@ -413,7 +414,7 @@ private Entry createEntry(String tableName, TreeMap scopes, byt WALEdit edit1 = new WALEdit(); for (byte[] kv : kvs) { - edit1.add(new KeyValue(kv, kv, kv)); + WALEditInternalHelper.addExtendedCell(edit1, new KeyValue(kv, kv, kv)); } return new Entry(key1, edit1); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java index aae2af10264b..06fdc47fa3ae 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java @@ -51,6 +51,7 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; +import org.apache.hadoop.hbase.wal.WALEditInternalHelper; import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.junit.Before; import org.junit.ClassRule; @@ -423,7 +424,8 @@ public void testReplicationInReplay() throws Exception { final byte[] value = Bytes.toBytes("v"); WALEdit edit = new WALEdit(true); long now = EnvironmentEdgeManager.currentTime(); - edit.add(new KeyValue(rowName, famName, qualifier, now, value)); + WALEditInternalHelper.addExtendedCell(edit, + new KeyValue(rowName, famName, qualifier, now, value)); WALKeyImpl walKey = new WALKeyImpl(hri.getEncodedNameAsBytes(), tableName, now, mvcc, scopes); wal.appendData(hri, walKey, edit); wal.sync(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWALEntryFilters.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWALEntryFilters.java index cd18443d80dd..fb4729d87f83 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWALEntryFilters.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWALEntryFilters.java @@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.wal.WAL.Entry; import org.apache.hadoop.hbase.wal.WALEdit; +import org.apache.hadoop.hbase.wal.WALEditInternalHelper; import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.junit.Assert; import org.junit.ClassRule; @@ -489,7 +490,7 @@ private Entry createEntry(TreeMap scopes, byte[]... kvs) { WALEdit edit1 = new WALEdit(); for (byte[] kv : kvs) { - edit1.add(new KeyValue(kv, kv, kv)); + WALEditInternalHelper.addExtendedCell(edit1, new KeyValue(kv, kv, kv)); } return new Entry(key1, edit1); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/master/TestRecoverStandbyProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/master/TestRecoverStandbyProcedure.java index 63f4f6ca0dbc..1b7b6c817495 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/master/TestRecoverStandbyProcedure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/master/TestRecoverStandbyProcedure.java @@ -54,6 +54,7 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.wal.WAL.Entry; import org.apache.hadoop.hbase.wal.WALEdit; +import org.apache.hadoop.hbase.wal.WALEditInternalHelper; import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.junit.After; import org.junit.AfterClass; @@ -183,7 +184,8 @@ private List setupWALEntries(int startRow, int endRow) { private Entry createWALEntry(byte[] row, byte[] value) { WALKeyImpl key = new WALKeyImpl(regionInfo.getEncodedNameAsBytes(), tableName, 1); WALEdit edit = new WALEdit(); - edit.add(new KeyValue(row, family, qualifier, timestamp, value)); + WALEditInternalHelper.addExtendedCell(edit, + new KeyValue(row, family, qualifier, timestamp, value)); return new Entry(key, edit); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestBasicWALEntryStream.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestBasicWALEntryStream.java index 991aa2db4d3d..ac21c6619f5f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestBasicWALEntryStream.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestBasicWALEntryStream.java @@ -68,6 +68,7 @@ import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; +import org.apache.hadoop.hbase.wal.WALEditInternalHelper; import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.wal.WALProvider; @@ -557,7 +558,7 @@ private void appendEntriesToLogAndSync(int count) throws IOException { private WALEdit getWALEdit(String row) { WALEdit edit = new WALEdit(); - edit.add(new KeyValue(Bytes.toBytes(row), family, qualifier, + WALEditInternalHelper.addExtendedCell(edit, new KeyValue(Bytes.toBytes(row), family, qualifier, EnvironmentEdgeManager.currentTime(), qualifier)); return edit; } @@ -700,7 +701,7 @@ private void appendEntries(WALProvider.Writer writer, int numEntries) throws IOE byte[] b = Bytes.toBytes(Integer.toString(i)); KeyValue kv = new KeyValue(b, b, b); WALEdit edit = new WALEdit(); - edit.add(kv); + WALEditInternalHelper.addExtendedCell(edit, kv); WALKeyImpl key = new WALKeyImpl(b, TableName.valueOf(b), 0, 0, HConstants.DEFAULT_CLUSTER_ID); NavigableMap scopes = new TreeMap(Bytes.BYTES_COMPARATOR); scopes.put(b, HConstants.REPLICATION_SCOPE_GLOBAL); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestHBaseInterClusterReplicationEndpointFilterEdits.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestHBaseInterClusterReplicationEndpointFilterEdits.java index cdef7de2076b..7b108f5ca148 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestHBaseInterClusterReplicationEndpointFilterEdits.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestHBaseInterClusterReplicationEndpointFilterEdits.java @@ -26,6 +26,7 @@ import java.util.List; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.ExtendedCell; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.KeyValue; @@ -40,6 +41,7 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.wal.WAL.Entry; import org.apache.hadoop.hbase.wal.WALEdit; +import org.apache.hadoop.hbase.wal.WALEditInternalHelper; import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -98,16 +100,16 @@ public static void tearDownAfterClass() throws Exception { public void testFilterNotExistColumnFamilyEdits() { List> entryList = new ArrayList<>(); // should be filtered - Cell c1 = new KeyValue(ROW, NON_EXISTING_FAMILY, QUALIFIER, + ExtendedCell c1 = new KeyValue(ROW, NON_EXISTING_FAMILY, QUALIFIER, EnvironmentEdgeManager.currentTime(), Type.Put, VALUE); Entry e1 = new Entry(new WALKeyImpl(new byte[32], TABLE1, EnvironmentEdgeManager.currentTime()), - new WALEdit().add(c1)); + WALEditInternalHelper.addExtendedCell(new WALEdit(), c1)); entryList.add(Lists.newArrayList(e1)); // should be kept - Cell c2 = + ExtendedCell c2 = new KeyValue(ROW, FAMILY, QUALIFIER, EnvironmentEdgeManager.currentTime(), Type.Put, VALUE); Entry e2 = new Entry(new WALKeyImpl(new byte[32], TABLE1, EnvironmentEdgeManager.currentTime()), - new WALEdit().add(c2)); + WALEditInternalHelper.addExtendedCell(new WALEdit(), c2)); entryList.add(Lists.newArrayList(e2, e1)); List> filtered = endpoint.filterNotExistColumnFamilyEdits(entryList); assertEquals(1, filtered.size()); @@ -120,16 +122,16 @@ public void testFilterNotExistColumnFamilyEdits() { public void testFilterNotExistTableEdits() { List> entryList = new ArrayList<>(); // should be filtered - Cell c1 = + ExtendedCell c1 = new KeyValue(ROW, FAMILY, QUALIFIER, EnvironmentEdgeManager.currentTime(), Type.Put, VALUE); Entry e1 = new Entry(new WALKeyImpl(new byte[32], TABLE2, EnvironmentEdgeManager.currentTime()), - new WALEdit().add(c1)); + WALEditInternalHelper.addExtendedCell(new WALEdit(), c1)); entryList.add(Lists.newArrayList(e1)); // should be kept - Cell c2 = + ExtendedCell c2 = new KeyValue(ROW, FAMILY, QUALIFIER, EnvironmentEdgeManager.currentTime(), Type.Put, VALUE); Entry e2 = new Entry(new WALKeyImpl(new byte[32], TABLE1, EnvironmentEdgeManager.currentTime()), - new WALEdit().add(c2)); + WALEditInternalHelper.addExtendedCell(new WALEdit(), c2)); entryList.add(Lists.newArrayList(e2)); List> filtered = endpoint.filterNotExistTableEdits(entryList); assertEquals(1, filtered.size()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSource.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSource.java index 05b268d3a0a9..37af52eb93b9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSource.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSource.java @@ -37,9 +37,9 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellBuilderFactory; import org.apache.hadoop.hbase.CellBuilderType; import org.apache.hadoop.hbase.CompatibilitySingletonFactory; +import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtil; @@ -66,6 +66,7 @@ import org.apache.hadoop.hbase.util.ManualEnvironmentEdge; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; +import org.apache.hadoop.hbase.wal.WALEditInternalHelper; import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.wal.WALProvider; @@ -186,9 +187,10 @@ public void testWALEntryFilter() throws IOException { TEST_UTIL.waitFor(30000, () -> rs.getWalEntryFilter() != null); WALEntryFilter wef = rs.getWalEntryFilter(); // Test non-system WAL edit. - WALEdit we = new WALEdit() - .add(CellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(HConstants.EMPTY_START_ROW) - .setFamily(HConstants.CATALOG_FAMILY).setType(Cell.Type.Put).build()); + WALEdit we = WALEditInternalHelper.addExtendedCell(new WALEdit(), + ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(HConstants.EMPTY_START_ROW).setFamily(HConstants.CATALOG_FAMILY) + .setType(Cell.Type.Put).build()); WAL.Entry e = new WAL.Entry( new WALKeyImpl(HConstants.EMPTY_BYTE_ARRAY, TableName.valueOf("test"), -1, -1, uuid), we); assertTrue(wef.filter(e) == e); @@ -222,7 +224,7 @@ public void testLogMoving() throws Exception { byte[] b = Bytes.toBytes(Integer.toString(i)); KeyValue kv = new KeyValue(b, b, b); WALEdit edit = new WALEdit(); - edit.add(kv); + WALEditInternalHelper.addExtendedCell(edit, kv); WALKeyImpl key = new WALKeyImpl(b, TableName.valueOf(b), 0, 0, HConstants.DEFAULT_CLUSTER_ID); writer.append(new WAL.Entry(key, edit)); writer.sync(false); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java index ffeb22d01bcc..663b444dc4e4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java @@ -36,9 +36,9 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellBuilderFactory; import org.apache.hadoop.hbase.CellBuilderType; import org.apache.hadoop.hbase.CompatibilitySingletonFactory; +import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.HConstants; @@ -66,6 +66,7 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; +import org.apache.hadoop.hbase.wal.WALEditInternalHelper; import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.hamcrest.Matchers; @@ -238,10 +239,12 @@ private void createWALFile(Path file) throws Exception { WALKeyImpl key = new WALKeyImpl(RI.getEncodedNameAsBytes(), TABLE_NAME, EnvironmentEdgeManager.currentTime(), SCOPES); WALEdit edit = new WALEdit(); - edit.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(F1).setFamily(F1) - .setQualifier(F1).setType(Cell.Type.Put).setValue(F1).build()); - edit.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(F2).setFamily(F2) - .setQualifier(F2).setType(Cell.Type.Put).setValue(F2).build()); + WALEditInternalHelper.addExtendedCell(edit, + ExtendedCellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(F1).setFamily(F1) + .setQualifier(F1).setType(Cell.Type.Put).setValue(F1).build()); + WALEditInternalHelper.addExtendedCell(edit, + ExtendedCellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(F2).setFamily(F2) + .setQualifier(F2).setType(Cell.Type.Put).setValue(F2).build()); writer.append(new WAL.Entry(key, edit)); writer.sync(false); } finally { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStreamCompressionReset.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStreamCompressionReset.java index 628ddcafeb08..aa30027b3c98 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStreamCompressionReset.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStreamCompressionReset.java @@ -32,8 +32,8 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellBuilderFactory; import org.apache.hadoop.hbase.CellBuilderType; +import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.HConstants; @@ -51,6 +51,7 @@ import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; +import org.apache.hadoop.hbase.wal.WALEditInternalHelper; import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -112,20 +113,23 @@ private static Pair generateWAL() throws Exception { writer.init(FS, path, UTIL.getConfiguration(), false, FS.getDefaultBlockSize(path), null); for (int i = 0; i < Byte.MAX_VALUE; i++) { WALEdit edit = new WALEdit(); - edit.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setType(Cell.Type.Put) - .setRow(Bytes.toBytes(i)).setFamily(FAMILY).setQualifier(Bytes.toBytes("qualifier-" + i)) - .setValue(Bytes.toBytes("v-" + i)).build()); + WALEditInternalHelper.addExtendedCell(edit, + ExtendedCellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setType(Cell.Type.Put) + .setRow(Bytes.toBytes(i)).setFamily(FAMILY).setQualifier(Bytes.toBytes("qualifier-" + i)) + .setValue(Bytes.toBytes("v-" + i)).build()); writer.append(new WAL.Entry(new WALKeyImpl(REGION_INFO.getEncodedNameAsBytes(), TABLE_NAME, EnvironmentEdgeManager.currentTime(), MVCC, SCOPE), edit)); } WALEdit edit2 = new WALEdit(); - edit2.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setType(Cell.Type.Put) - .setRow(Bytes.toBytes(-1)).setFamily(FAMILY).setQualifier(Bytes.toBytes("qualifier")) - .setValue(Bytes.toBytes("vv")).build()); - edit2.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setType(Cell.Type.Put) - .setRow(Bytes.toBytes(-1)).setFamily(FAMILY).setQualifier(Bytes.toBytes("qualifier-1")) - .setValue(Bytes.toBytes("vvv")).build()); + WALEditInternalHelper.addExtendedCell(edit2, + ExtendedCellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setType(Cell.Type.Put) + .setRow(Bytes.toBytes(-1)).setFamily(FAMILY).setQualifier(Bytes.toBytes("qualifier")) + .setValue(Bytes.toBytes("vv")).build()); + WALEditInternalHelper.addExtendedCell(edit2, + ExtendedCellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setType(Cell.Type.Put) + .setRow(Bytes.toBytes(-1)).setFamily(FAMILY).setQualifier(Bytes.toBytes("qualifier-1")) + .setValue(Bytes.toBytes("vvv")).build()); writer.append(new WAL.Entry(new WALKeyImpl(REGION_INFO.getEncodedNameAsBytes(), TABLE_NAME, EnvironmentEdgeManager.currentTime(), MVCC, SCOPE), edit2)); writer.sync(false); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/WALEntryStreamTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/WALEntryStreamTestBase.java index be1c66c815c8..3af5596b0054 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/WALEntryStreamTestBase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/WALEntryStreamTestBase.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; +import org.apache.hadoop.hbase.wal.WALEditInternalHelper; import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hdfs.DistributedFileSystem; @@ -177,8 +178,9 @@ protected long appendToLog(int count) throws IOException { protected WALEdit getWALEdits(int count) { WALEdit edit = new WALEdit(); for (int i = 0; i < count; i++) { - edit.add(new KeyValue(Bytes.toBytes(EnvironmentEdgeManager.currentTime()), family, qualifier, - EnvironmentEdgeManager.currentTime(), qualifier)); + WALEditInternalHelper.addExtendedCell(edit, + new KeyValue(Bytes.toBytes(EnvironmentEdgeManager.currentTime()), family, qualifier, + EnvironmentEdgeManager.currentTime(), qualifier)); } return edit; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/CompressedWALTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/CompressedWALTestBase.java index 2ff9223e74b3..93714111d8da 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/CompressedWALTestBase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/CompressedWALTestBase.java @@ -28,8 +28,8 @@ import java.util.TreeMap; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellBuilderFactory; import org.apache.hadoop.hbase.CellBuilderType; +import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionInfo; @@ -111,9 +111,9 @@ public void doTest(TableName tableName, int valueSize) throws Exception { for (int i = 0; i < total; i++) { WALEdit kvs = new WALEdit(); - kvs.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setType(Cell.Type.Put) + kvs.add(ExtendedCellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setType(Cell.Type.Put) .setRow(row).setFamily(family).setQualifier(Bytes.toBytes(i)).setValue(value).build()); - kvs.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY) + kvs.add(ExtendedCellBuilderFactory.create(CellBuilderType.SHALLOW_COPY) .setType(Cell.Type.DeleteFamily).setRow(row).setFamily(family).build()); wal.appendData(regionInfo, new WALKeyImpl(regionInfo.getEncodedNameAsBytes(), tableName, System.currentTimeMillis(), mvcc, scopes), kvs); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestParsePartialWALFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestParsePartialWALFile.java index 108bcd8f8b4e..dc267058183e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestParsePartialWALFile.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestParsePartialWALFile.java @@ -31,9 +31,9 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.Cell.Type; -import org.apache.hadoop.hbase.CellBuilderFactory; import org.apache.hadoop.hbase.CellBuilderType; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtil; import org.apache.hadoop.hbase.HConstants; @@ -157,10 +157,10 @@ public void testPartialParse() throws Exception { EnvironmentEdgeManager.currentTime(), HConstants.DEFAULT_CLUSTER_ID); WALEdit edit = new WALEdit(); if (i % 2 == 0) { - edit.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setType(Type.Put) + edit.add(ExtendedCellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setType(Type.Put) .setRow(ROW).setFamily(FAMILY).setQualifier(QUAL).setValue(VALUE).build()); } else { - edit.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY) + edit.add(ExtendedCellBuilderFactory.create(CellBuilderType.SHALLOW_COPY) .setType(Type.DeleteFamily).setRow(ROW).setFamily(FAMILY).build()); } writer.append(new WAL.Entry(key, edit)); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java index c462e2e5c621..97ad666c1b18 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java @@ -47,6 +47,7 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.MockRegionServerServices; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ClientInternalHelper; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionInfo; @@ -152,7 +153,7 @@ public void run() { long now = System.nanoTime(); Put put = setupPut(ThreadLocalRandom.current(), key, value, numFamilies); WALEdit walEdit = new WALEdit(); - walEdit.add(put.getFamilyCellMap()); + walEdit.addMap(ClientInternalHelper.getExtendedFamilyCellMap(put)); RegionInfo hri = region.getRegionInfo(); final WALKeyImpl logkey = new WALKeyImpl(hri.getEncodedNameAsBytes(), hri.getTable(), now, mvcc, scopes); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java index ee060bd53878..c802883c9391 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java @@ -45,6 +45,7 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Append; +import org.apache.hadoop.hbase.client.ClientInternalHelper; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Consistency; @@ -56,7 +57,6 @@ import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.OnlineLogRecord; import org.apache.hadoop.hbase.client.OperationWithAttributes; -import org.apache.hadoop.hbase.client.PackagePrivateFieldAccessor; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.Result; @@ -222,7 +222,7 @@ public static List getsFromThrift(List in) throws IOException { * @return converted result, returns an empty result if the input is null */ public static TResult resultFromHBase(Result in) { - ExtendedCell[] raw = PackagePrivateFieldAccessor.getExtendedRawCells(in); + ExtendedCell[] raw = ClientInternalHelper.getExtendedRawCells(in); TResult out = new TResult(); byte[] row = in.getRow(); if (row != null) { @@ -1311,7 +1311,7 @@ public static TPut putFromHBase(Put in) { if (in.getDurability() != Durability.USE_DEFAULT) { out.setDurability(durabilityFromHBase(in.getDurability())); } - for (Map.Entry> entry : PackagePrivateFieldAccessor + for (Map.Entry> entry : ClientInternalHelper .getExtendedFamilyCellMap(in).entrySet()) { byte[] family = entry.getKey(); for (ExtendedCell cell : entry.getValue()) { @@ -1375,7 +1375,7 @@ public static TAppend appendFromHBase(Append in) throws IOException { if (in.getDurability() != Durability.USE_DEFAULT) { out.setDurability(durabilityFromHBase(in.getDurability())); } - for (Map.Entry> entry : PackagePrivateFieldAccessor + for (Map.Entry> entry : ClientInternalHelper .getExtendedFamilyCellMap(in).entrySet()) { byte[] family = entry.getKey(); for (ExtendedCell cell : entry.getValue()) {