From eda23a5e3dcbc596b31556b1c992dfb67bca3c4c Mon Sep 17 00:00:00 2001 From: LiangJun He <2005hithlj@163.com> Date: Tue, 24 May 2022 12:06:59 +0800 Subject: [PATCH 1/4] HBASE-27028 Add a shell command for flushing master local region --- .../org/apache/hadoop/hbase/client/Admin.java | 5 +++ .../hbase/client/AdminOverAsyncAdmin.java | 5 +++ .../hadoop/hbase/client/AsyncAdmin.java | 5 +++ .../hadoop/hbase/client/AsyncHBaseAdmin.java | 5 +++ .../hbase/client/RawAsyncHBaseAdmin.java | 12 ++++++ .../main/protobuf/server/master/Master.proto | 6 +++ .../apache/hadoop/hbase/master/HMaster.java | 16 ++++++++ .../hbase/master/MasterRpcServices.java | 14 +++++++ .../hadoop/hbase/master/MasterServices.java | 5 +++ .../hbase/master/region/MasterRegion.java | 3 +- .../MasterRegionFlusherAndCompactor.java | 6 ++- .../hbase/master/MockNoopMasterServices.java | 4 ++ .../master/region/TestMasterRegionFlush.java | 10 +++++ .../hbase/rsgroup/VerifyingRSGroupAdmin.java | 5 +++ hbase-shell/src/main/ruby/hbase/admin.rb | 8 +++- hbase-shell/src/main/ruby/shell.rb | 1 + .../ruby/shell/commands/flush_master_store.rb | 37 +++++++++++++++++++ .../hbase/thrift2/client/ThriftAdmin.java | 5 +++ 18 files changed, 148 insertions(+), 4 deletions(-) create mode 100644 hbase-shell/src/main/ruby/shell/commands/flush_master_store.rb diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java index 736daf93eff2..d2b57476128c 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java @@ -2539,4 +2539,9 @@ default BalanceResponse balanceRSGroup(String groupName) throws IOException { */ List getLogEntries(Set serverNames, String logType, ServerType serverType, int limit, Map filterParams) throws IOException; + + /** + * Flush master local region + */ + void flushMasterStore() throws IOException; } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java index 901c86ff3744..efab24904622 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java @@ -1078,4 +1078,9 @@ public List getLogEntries(Set serverNames, String logType, ServerType serverType, int limit, Map filterParams) throws IOException { return get(admin.getLogEntries(serverNames, logType, serverType, limit, filterParams)); } + + @Override + public void flushMasterStore() throws IOException { + get(admin.flushMasterStore()); + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java index 47b89c96a4b3..e0a254bb2aa4 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java @@ -1785,4 +1785,9 @@ default CompletableFuture balanceRSGroup(String groupName) { */ CompletableFuture> getLogEntries(Set serverNames, String logType, ServerType serverType, int limit, Map filterParams); + + /** + * Flush master local region + */ + CompletableFuture flushMasterStore(); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java index 7b44c2d341d2..ba016111cd3d 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java @@ -949,4 +949,9 @@ public CompletableFuture> getLogEntries(Set serverNam String logType, ServerType serverType, int limit, Map filterParams) { return wrap(rawAdmin.getLogEntries(serverNames, logType, serverType, limit, filterParams)); } + + @Override + public CompletableFuture flushMasterStore() { + return wrap(rawAdmin.flushMasterStore()); + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java index d88279eeafc6..3ec2c741293e 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java @@ -178,6 +178,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.EnableTableResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ExecProcedureRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ExecProcedureResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.FlushMasterStoreRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.FlushMasterStoreResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetClusterStatusRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetClusterStatusResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetCompletedSnapshotsRequest; @@ -4280,4 +4282,14 @@ public CompletableFuture> getLogEntries(Set serverNam return CompletableFuture.completedFuture(Collections.emptyList()); } } + + @Override + public CompletableFuture flushMasterStore() { + FlushMasterStoreRequest.Builder request = FlushMasterStoreRequest.newBuilder(); + return this. newMasterCaller() + .action(((controller, stub) -> this. call(controller, stub, request.build(), + (s, c, req, done) -> s.flushMasterStore(c, req, done), resp -> null))) + .call(); + } } diff --git a/hbase-protocol-shaded/src/main/protobuf/server/master/Master.proto b/hbase-protocol-shaded/src/main/protobuf/server/master/Master.proto index 94a434755cff..257abe8f11ca 100644 --- a/hbase-protocol-shaded/src/main/protobuf/server/master/Master.proto +++ b/hbase-protocol-shaded/src/main/protobuf/server/master/Master.proto @@ -758,6 +758,9 @@ message ModifyColumnStoreFileTrackerResponse { optional uint64 proc_id = 1; } +message FlushMasterStoreRequest {} +message FlushMasterStoreResponse {} + service MasterService { /** Used by the client to get the number of regions that have received the updated schema */ rpc GetSchemaAlterStatus(GetSchemaAlterStatusRequest) @@ -1197,6 +1200,9 @@ service MasterService { rpc ModifyColumnStoreFileTracker(ModifyColumnStoreFileTrackerRequest) returns(ModifyColumnStoreFileTrackerResponse); + + rpc FlushMasterStore(FlushMasterStoreRequest) + returns(FlushMasterStoreResponse); } // HBCK Service definitions. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 42367857db10..7f98bf4dbfe2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -457,6 +457,9 @@ public class HMaster extends HBaseServerBase implements Maste public static final String WARMUP_BEFORE_MOVE = "hbase.master.warmup.before.move"; private static final boolean DEFAULT_WARMUP_BEFORE_MOVE = true; + // Only for testing + private boolean isLocalRegionFlushed = false; + /** * Initializes the HMaster. The steps are as follows: *

@@ -4207,6 +4210,19 @@ public List getMetaLocations() { return metaRegionLocationCache.getMetaRegionLocations(); } + @Override + public void flushMasterStore() throws IOException { + LOG.info("Force flush master local region."); + masterRegion.flush(true); + isLocalRegionFlushed = true; + } + + @RestrictedApi(explanation = "Should only be called in tests", link = "", + allowedOnPath = ".*/src/test/.*") + public boolean isLocalRegionFlushed() { + return isLocalRegionFlushed; + } + public Collection getLiveRegionServers() { return regionServerTracker.getRegionServers(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index fd1826fd9cad..a22b7a620c04 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -229,6 +229,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ExecProcedureResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.FixMetaRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.FixMetaResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.FlushMasterStoreRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.FlushMasterStoreResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetClusterStatusRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetClusterStatusResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetCompletedSnapshotsRequest; @@ -3468,4 +3470,16 @@ public ReplicateWALEntryResponse replicateToReplica(RpcController controller, ReplicateWALEntryRequest request) throws ServiceException { throw new ServiceException(new DoNotRetryIOException("Unsupported method on master")); } + + @Override + public FlushMasterStoreResponse flushMasterStore(RpcController controller, + FlushMasterStoreRequest request) throws ServiceException { + rpcPreCheck("flushMasterStore"); + try { + server.flushMasterStore(); + } catch (IOException ioe) { + throw new ServiceException(ioe); + } + return FlushMasterStoreResponse.newBuilder().build(); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java index 8aaf2c28d599..d813b39863ef 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java @@ -510,4 +510,9 @@ boolean normalizeRegions(final NormalizeTableFilterParams ntfp, final boolean is * We need to get this in MTP to tell the syncer the new meta replica count. */ MetaLocationSyncer getMetaLocationSyncer(); + + /** + * Flush master local region + */ + void flushMasterStore() throws IOException; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java index b8bdc1250afd..c3b5e5d42646 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java @@ -161,9 +161,8 @@ public RegionScanner getRegionScanner(Scan scan) throws IOException { return region.getScanner(scan); } - @RestrictedApi(explanation = "Should only be called in tests", link = "", - allowedOnPath = ".*/src/test/.*") public FlushResult flush(boolean force) throws IOException { + flusherAndCompactor.resetChangesAfterLastFlush(); return region.flush(force); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionFlusherAndCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionFlusherAndCompactor.java index 2e970ae4b93f..b0477d0fea74 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionFlusherAndCompactor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionFlusherAndCompactor.java @@ -202,7 +202,7 @@ private void flushLoop() { flushLock.unlock(); } assert flushRequest; - changesAfterLastFlush.set(0); + resetChangesAfterLastFlush(); try { region.flush(true); lastFlushTime = EnvironmentEdgeManager.currentTime(); @@ -263,6 +263,10 @@ void requestFlush() { } } + void resetChangesAfterLastFlush() { + changesAfterLastFlush.set(0); + } + @Override public void close() { closed = true; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java index 6a42f66e9278..bc3969ffd518 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java @@ -503,6 +503,10 @@ public MetaLocationSyncer getMetaLocationSyncer() { return null; } + @Override + public void flushMasterStore() { + } + @Override public long modifyTableStoreFileTracker(TableName tableName, String dstSFT, long nonceGroup, long nonce) throws IOException { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/region/TestMasterRegionFlush.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/region/TestMasterRegionFlush.java index 4a8933d0031d..e5c78103d47f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/region/TestMasterRegionFlush.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/region/TestMasterRegionFlush.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.client.RegionInfoBuilder; @@ -148,6 +149,15 @@ public void testPeriodicalFlush() throws InterruptedException { assertEquals(1, flushCalled.get()); Thread.sleep(1000); assertEquals(2, flushCalled.get()); + } + @Test + public void testForceFlush() throws Exception { + // Test for HBASE-27028 + HBaseTestingUtil htu = new HBaseTestingUtil(); + htu.startMiniCluster(1); + htu.getConnection().getAdmin().flushMasterStore(); + assertTrue(htu.getMiniHBaseCluster().getMaster().isLocalRegionFlushed()); + htu.shutdownMiniCluster(); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java index 27aabe51743f..3c0658455f3a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java @@ -934,4 +934,9 @@ public Future modifyTableStoreFileTrackerAsync(TableName tableName, String throws IOException { return admin.modifyTableStoreFileTrackerAsync(tableName, dstSFT); } + + @Override + public void flushMasterStore() throws IOException { + admin.flushMasterStore(); + } } diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index 9952cff60d66..e90e37c4bc63 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -1868,11 +1868,17 @@ def modify_table_sft(tableName, sft) @admin.modifyTableStoreFileTracker(tableName, sft) end - #---------------------------------------------------------------------------------------------- + #---------------------------------------------------------------------------------------------- # Change table column family's sft def modify_table_family_sft(tableName, family_bytes, sft) @admin.modifyColumnFamilyStoreFileTracker(tableName, family_bytes, sft) end + + #---------------------------------------------------------------------------------------------- + # Flush master local region + def flush_master_store() + @admin.flushMasterStore() + end end # rubocop:enable Metrics/ClassLength end diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb index 38a6791015fc..4eb629b8cc1b 100644 --- a/hbase-shell/src/main/ruby/shell.rb +++ b/hbase-shell/src/main/ruby/shell.rb @@ -446,6 +446,7 @@ def self.exception_handler(hide_traceback) compact compaction_switch flush + flush_master_store get_balancer_decisions get_balancer_rejections get_slowlog_responses diff --git a/hbase-shell/src/main/ruby/shell/commands/flush_master_store.rb b/hbase-shell/src/main/ruby/shell/commands/flush_master_store.rb new file mode 100644 index 000000000000..91b8961fffa7 --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/flush_master_store.rb @@ -0,0 +1,37 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class FlushMasterStore < Command + def help + <<-EOF +Flush master local region. +For example: + + hbase> flush_master_store +EOF + end + + def command() + admin.flush_master_store() + end + end + end +end diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java index 3c7586f852d2..13a1b9920ecf 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java @@ -1309,4 +1309,9 @@ public Future modifyTableStoreFileTrackerAsync(TableName tableName, String throw new NotImplementedException( "modifyTableStoreFileTrackerAsync not supported in ThriftAdmin"); } + + @Override + public void flushMasterStore() throws IOException { + throw new NotImplementedException("flushMasterStore not supported in ThriftAdmin"); + } } From 8718a5e2a44924230f4baf822d5cefed08fd600b Mon Sep 17 00:00:00 2001 From: LiangJun He <2005hithlj@163.com> Date: Wed, 8 Jun 2022 00:46:51 +0800 Subject: [PATCH 2/4] HBASE-27028 Add a shell command for flushing master local region --- .../hbase/coprocessor/MasterObserver.java | 16 +++++++++ .../apache/hadoop/hbase/master/HMaster.java | 24 +++++++------ .../hbase/master/MasterCoprocessorHost.java | 18 ++++++++++ .../hbase/coprocessor/TestMasterObserver.java | 35 +++++++++++++++++++ .../master/region/TestMasterRegionFlush.java | 10 ------ 5 files changed, 83 insertions(+), 20 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java index 8a7cd9cfc6b3..6f6cb35835b0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java @@ -1000,6 +1000,22 @@ default void postTableFlush(final ObserverContext final TableName tableName) throws IOException { } + /** + * Called before the master local region memstore is flushed to disk. + * @param ctx the environment to interact with the framework and master + */ + default void preMasterStoreFlush(final ObserverContext ctx) + throws IOException { + } + + /** + * Called after the master local region memstore is flushed to disk. + * @param ctx the environment to interact with the framework and master + */ + default void postMasterStoreFlush(final ObserverContext ctx) + throws IOException { + } + /** * Called before the quota for the user is stored. * @param ctx the environment to interact with the framework and master diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 7f98bf4dbfe2..a2a86c4251da 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -457,9 +457,6 @@ public class HMaster extends HBaseServerBase implements Maste public static final String WARMUP_BEFORE_MOVE = "hbase.master.warmup.before.move"; private static final boolean DEFAULT_WARMUP_BEFORE_MOVE = true; - // Only for testing - private boolean isLocalRegionFlushed = false; - /** * Initializes the HMaster. The steps are as follows: *

@@ -4213,14 +4210,21 @@ public List getMetaLocations() { @Override public void flushMasterStore() throws IOException { LOG.info("Force flush master local region."); + if (this.cpHost != null) { + try { + cpHost.preMasterStoreFlush(); + } catch (IOException ioe) { + LOG.error("Error invoking master coprocessor preMasterStoreFlush()", ioe); + } + } masterRegion.flush(true); - isLocalRegionFlushed = true; - } - - @RestrictedApi(explanation = "Should only be called in tests", link = "", - allowedOnPath = ".*/src/test/.*") - public boolean isLocalRegionFlushed() { - return isLocalRegionFlushed; + if (this.cpHost != null) { + try { + cpHost.postMasterStoreFlush(); + } catch (IOException ioe) { + LOG.error("Error invoking master coprocessor postMasterStoreFlush()", ioe); + } + } } public Collection getLiveRegionServers() { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java index 4a1aeb652fa8..6295fa63d501 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java @@ -1218,6 +1218,24 @@ public void call(MasterObserver observer) throws IOException { }); } + public void preMasterStoreFlush() throws IOException { + execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() { + @Override + public void call(MasterObserver observer) throws IOException { + observer.preMasterStoreFlush(this); + } + }); + } + + public void postMasterStoreFlush() throws IOException { + execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() { + @Override + public void call(MasterObserver observer) throws IOException { + observer.postMasterStoreFlush(this); + } + }); + } + public void preSetUserQuota(final String user, final GlobalQuotaSettings quotas) throws IOException { execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java index bc7535cde9af..9e65e58a56d2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java @@ -189,6 +189,8 @@ public static class CPMasterObserver implements MasterCoprocessor, MasterObserve private boolean postRequestLockCalled; private boolean preLockHeartbeatCalled; private boolean postLockHeartbeatCalled; + private boolean preMasterStoreFlushCalled; + private boolean postMasterStoreFlushCalled; public void resetStates() { preCreateTableRegionInfosCalled = false; @@ -280,6 +282,8 @@ public void resetStates() { postRequestLockCalled = false; preLockHeartbeatCalled = false; postLockHeartbeatCalled = false; + preMasterStoreFlushCalled = false; + postMasterStoreFlushCalled = false; } @Override @@ -1045,6 +1049,18 @@ public void postTableFlush(ObserverContext ctx, TableName tableName) throws IOException { } + @Override + public void preMasterStoreFlush(ObserverContext ctx) + throws IOException { + preMasterStoreFlushCalled = true; + } + + @Override + public void postMasterStoreFlush(ObserverContext ctx) + throws IOException { + postMasterStoreFlushCalled = true; + } + @Override public void preSetUserQuota(final ObserverContext ctx, final String userName, final GlobalQuotaSettings quotas) throws IOException { @@ -1680,4 +1696,23 @@ public void testQueueLockAndLockHeartbeatOperations() throws Exception { ProcedureTestingUtility.waitNoProcedureRunning(master.getMasterProcedureExecutor()); ProcedureTestingUtility.assertProcNotFailed(master.getMasterProcedureExecutor(), procId); } + + @Test + public void testMasterStoreOperations() throws Exception { + SingleProcessHBaseCluster cluster = UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + MasterCoprocessorHost host = master.getMasterCoprocessorHost(); + CPMasterObserver cp = host.findCoprocessor(CPMasterObserver.class); + cp.resetStates(); + assertFalse("No master store flush call", cp.preMasterStoreFlushCalled); + assertFalse("No master store flush call", cp.postMasterStoreFlushCalled); + + try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration()); + Admin admin = connection.getAdmin()) { + admin.flushMasterStore(); + + assertTrue("Master store flush called", cp.preMasterStoreFlushCalled); + assertTrue("Master store flush called", cp.postMasterStoreFlushCalled); + } + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/region/TestMasterRegionFlush.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/region/TestMasterRegionFlush.java index e5c78103d47f..e37dce48382e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/region/TestMasterRegionFlush.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/region/TestMasterRegionFlush.java @@ -150,14 +150,4 @@ public void testPeriodicalFlush() throws InterruptedException { Thread.sleep(1000); assertEquals(2, flushCalled.get()); } - - @Test - public void testForceFlush() throws Exception { - // Test for HBASE-27028 - HBaseTestingUtil htu = new HBaseTestingUtil(); - htu.startMiniCluster(1); - htu.getConnection().getAdmin().flushMasterStore(); - assertTrue(htu.getMiniHBaseCluster().getMaster().isLocalRegionFlushed()); - htu.shutdownMiniCluster(); - } } From 638621de32f6351c6d3191f9d8054e5b1e78c230 Mon Sep 17 00:00:00 2001 From: LiangJun He <2005hithlj@163.com> Date: Wed, 8 Jun 2022 00:54:42 +0800 Subject: [PATCH 3/4] HBASE-27028 Add a shell command for flushing master local region --- .../apache/hadoop/hbase/coprocessor/MasterObserver.java | 4 ++-- .../apache/hadoop/hbase/master/region/MasterRegion.java | 4 +++- .../master/region/MasterRegionFlusherAndCompactor.java | 8 ++++++-- .../hadoop/hbase/master/region/TestMasterRegionFlush.java | 1 - 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java index 6f6cb35835b0..ad381dd4ef38 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java @@ -1002,7 +1002,7 @@ default void postTableFlush(final ObserverContext /** * Called before the master local region memstore is flushed to disk. - * @param ctx the environment to interact with the framework and master + * @param ctx the environment to interact with the framework and master */ default void preMasterStoreFlush(final ObserverContext ctx) throws IOException { @@ -1010,7 +1010,7 @@ default void preMasterStoreFlush(final ObserverContext ctx) throws IOException { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java index c3b5e5d42646..a5a6ad39f244 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java @@ -163,7 +163,9 @@ public RegionScanner getRegionScanner(Scan scan) throws IOException { public FlushResult flush(boolean force) throws IOException { flusherAndCompactor.resetChangesAfterLastFlush(); - return region.flush(force); + FlushResult flushResult = region.flush(force); + flusherAndCompactor.resetLastFlushTime(); + return flushResult; } @RestrictedApi(explanation = "Should only be called in tests", link = "", diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionFlusherAndCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionFlusherAndCompactor.java index b0477d0fea74..474d87f370ff 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionFlusherAndCompactor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionFlusherAndCompactor.java @@ -180,7 +180,7 @@ private boolean needCompaction() { } private void flushLoop() { - lastFlushTime = EnvironmentEdgeManager.currentTime(); + resetLastFlushTime(); while (!closed) { flushLock.lock(); try { @@ -205,7 +205,7 @@ private void flushLoop() { resetChangesAfterLastFlush(); try { region.flush(true); - lastFlushTime = EnvironmentEdgeManager.currentTime(); + resetLastFlushTime(); } catch (IOException e) { LOG.error(HBaseMarkers.FATAL, "Failed to flush master local region, aborting...", e); abortable.abort("Failed to flush master local region", e); @@ -267,6 +267,10 @@ void resetChangesAfterLastFlush() { changesAfterLastFlush.set(0); } + void resetLastFlushTime() { + lastFlushTime = EnvironmentEdgeManager.currentTime(); + } + @Override public void close() { closed = true; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/region/TestMasterRegionFlush.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/region/TestMasterRegionFlush.java index e37dce48382e..b0f1a2027790 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/region/TestMasterRegionFlush.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/region/TestMasterRegionFlush.java @@ -33,7 +33,6 @@ import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.client.RegionInfoBuilder; From 83b4092067aab0c53e52ed0d5124ec5dfadca82a Mon Sep 17 00:00:00 2001 From: LiangJun He <2005hithlj@163.com> Date: Fri, 10 Jun 2022 18:02:31 +0800 Subject: [PATCH 4/4] HBASE-27028 Add a shell command for flushing master local region --- .../org/apache/hadoop/hbase/master/region/MasterRegion.java | 2 +- .../master/region/MasterRegionFlusherAndCompactor.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java index a5a6ad39f244..5ed0df0aa580 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java @@ -164,7 +164,7 @@ public RegionScanner getRegionScanner(Scan scan) throws IOException { public FlushResult flush(boolean force) throws IOException { flusherAndCompactor.resetChangesAfterLastFlush(); FlushResult flushResult = region.flush(force); - flusherAndCompactor.resetLastFlushTime(); + flusherAndCompactor.recordLastFlushTime(); return flushResult; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionFlusherAndCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionFlusherAndCompactor.java index 474d87f370ff..3d4bfea146e5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionFlusherAndCompactor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionFlusherAndCompactor.java @@ -180,7 +180,7 @@ private boolean needCompaction() { } private void flushLoop() { - resetLastFlushTime(); + recordLastFlushTime(); while (!closed) { flushLock.lock(); try { @@ -205,7 +205,7 @@ private void flushLoop() { resetChangesAfterLastFlush(); try { region.flush(true); - resetLastFlushTime(); + recordLastFlushTime(); } catch (IOException e) { LOG.error(HBaseMarkers.FATAL, "Failed to flush master local region, aborting...", e); abortable.abort("Failed to flush master local region", e); @@ -267,7 +267,7 @@ void resetChangesAfterLastFlush() { changesAfterLastFlush.set(0); } - void resetLastFlushTime() { + void recordLastFlushTime() { lastFlushTime = EnvironmentEdgeManager.currentTime(); }