diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java index 155f0249daa2..f93861e4e281 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java @@ -1322,6 +1322,14 @@ Future abortProcedureAsync(long procId, boolean mayInterruptIfRunning) */ void rollWALWriter(ServerName serverName) throws IOException, FailedLogCloseException; + /** + * Archive the earliest log file. + * + * @param serverName The servername of the regionserver. + * @throws IOException if a remote or network exception occurs + */ + void archiveWAL(ServerName serverName) throws IOException; + /** * Helper that delegates to getClusterMetrics().getMasterCoprocessorNames(). * @return an array of master coprocessors diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java index 1255753f05e0..4fab06c7df36 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java @@ -588,6 +588,11 @@ public void rollWALWriter(ServerName serverName) throws IOException, FailedLogCl get(admin.rollWALWriter(serverName)); } + @Override + public void archiveWAL(ServerName serverName) throws IOException { + get(admin.archiveWAL(serverName)); + } + @Override public CompactionState getCompactionState(TableName tableName) throws IOException { return get(admin.getCompactionState(tableName)); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java index 336903d42e38..88433db080b0 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java @@ -1127,6 +1127,13 @@ default CompletableFuture getMasterInfoPort() { */ CompletableFuture rollWALWriter(ServerName serverName); + /** + * Archive the earliest log file. + * + * @param serverName The servername of the region server. + */ + CompletableFuture archiveWAL(ServerName serverName); + /** * Clear compacting queues on a region server. * @param serverName diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java index c29fe7118dd7..5cde7403c66d 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java @@ -635,6 +635,11 @@ public CompletableFuture rollWALWriter(ServerName serverName) { return wrap(rawAdmin.rollWALWriter(serverName)); } + @Override + public CompletableFuture archiveWAL(ServerName serverName) { + return wrap(rawAdmin.archiveWAL(serverName)); + } + @Override public CompletableFuture clearCompactionQueues(ServerName serverName, Set queues) { return wrap(rawAdmin.clearCompactionQueues(serverName, queues)); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java index 1330b2a17562..347106a929c9 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java @@ -121,6 +121,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.AccessControlProtos.RevokeRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AccessControlProtos.RevokeResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ArchiveWALRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ArchiveWALResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearRegionBlockCacheRequest; @@ -2913,6 +2915,17 @@ public CompletableFuture rollWALWriter(ServerName serverName) { .serverName(serverName).call(); } + @Override + public CompletableFuture archiveWAL(ServerName serverName) { + return this + . newAdminCaller() + .action( + (controller, stub) -> this. adminCall( + controller, stub, RequestConverter.buildArchiveWALRequest(), + (s, c, req, done) -> s.archiveWAL(controller, req, done), resp -> null)) + .serverName(serverName).call(); + } + @Override public CompletableFuture clearCompactionQueues(ServerName serverName, Set queues) { return this diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java index 7b0282afc080..c9cf2a3bb9a4 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java @@ -72,6 +72,7 @@ import org.apache.hbase.thirdparty.com.google.common.collect.Sets; import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ArchiveWALRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearRegionBlockCacheRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearSlowLogResponseRequest; @@ -883,6 +884,20 @@ public static RollWALWriterRequest buildRollWALWriterRequest() { return ROLL_WAL_WRITER_REQUEST; } + /** + * @see #buildArchiveWALRequest() + */ + private static ArchiveWALRequest ARCHIVE_WAL_REQUEST = ArchiveWALRequest.newBuilder() + .build(); + + /** + * Get the static ArchiveWALRequest instance + * @return a ArchiveWALRequest + */ + public static ArchiveWALRequest buildArchiveWALRequest() { + return ARCHIVE_WAL_REQUEST; + } + /** * @see #buildGetServerInfoRequest() */ diff --git a/hbase-protocol-shaded/src/main/protobuf/server/region/Admin.proto b/hbase-protocol-shaded/src/main/protobuf/server/region/Admin.proto index 101ed1e3f6d5..739b600cf69e 100644 --- a/hbase-protocol-shaded/src/main/protobuf/server/region/Admin.proto +++ b/hbase-protocol-shaded/src/main/protobuf/server/region/Admin.proto @@ -219,6 +219,12 @@ message RollWALWriterResponse { repeated bytes region_to_flush = 1; } +message ArchiveWALRequest { +} + +message ArchiveWALResponse { +} + message StopServerRequest { required string reason = 1; } @@ -351,6 +357,9 @@ service AdminService { rpc RollWALWriter(RollWALWriterRequest) returns(RollWALWriterResponse); + rpc ArchiveWAL(ArchiveWALRequest) + returns(ArchiveWALResponse); + rpc GetServerInfo(GetServerInfoRequest) returns(GetServerInfoResponse); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java index f3ccd9d3638b..397baa152296 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java @@ -76,6 +76,24 @@ default void postRollWALWriterRequest( final ObserverContext ctx) throws IOException {} + /** + * This will be called before executing user request to archive earliest log + * file of a region server. + * @param ctx the environment to interact with the framework and region server. + */ + default void preArchiveWALRequest( + final ObserverContext ctx) + throws IOException {} + + /** + * This will be called after executing user request to archive earliest log + * file of a region server. + * @param ctx the environment to interact with the framework and region server. + */ + default void postArchiveWALRequest( + final ObserverContext ctx) + throws IOException {} + /** * This will be called after the replication endpoint is instantiated. * @param ctx the environment to interact with the framework and region server. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java index 4c0b62115d14..8c9e944059d1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java @@ -171,6 +171,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter; import org.apache.hadoop.hbase.shaded.protobuf.ResponseConverter; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ArchiveWALRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ArchiveWALResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearRegionBlockCacheRequest; @@ -2394,6 +2396,27 @@ public RollWALWriterResponse rollWALWriter(final RpcController controller, } } + /** + * Archive the earliest log file of the region server. + * @param controller the RPC controller + * @param request the request + * @throws ServiceException + */ + @Override + public ArchiveWALResponse archiveWAL(final RpcController controller, + final ArchiveWALRequest request) throws ServiceException { + try { + checkOpen(); + requestCount.increment(); + regionServer.getRegionServerCoprocessorHost().preArchiveWALRequest(); + regionServer.getWalRoller().requestArchive(); + regionServer.getRegionServerCoprocessorHost().postArchiveWALRequest(); + ArchiveWALResponse.Builder builder = ArchiveWALResponse.newBuilder(); + return builder.build(); + } catch (IOException ie) { + throw new ServiceException(ie); + } + } /** * Stop the region server. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java index ceb30ff21f47..2cfecce4f155 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java @@ -150,6 +150,24 @@ public void call(RegionServerObserver observer) throws IOException { }); } + public void preArchiveWALRequest() throws IOException { + execOperation(coprocEnvironments.isEmpty() ? null : new RegionServerObserverOperation() { + @Override + public void call(RegionServerObserver observer) throws IOException { + observer.preArchiveWALRequest(this); + } + }); + } + + public void postArchiveWALRequest() throws IOException { + execOperation(coprocEnvironments.isEmpty() ? null : new RegionServerObserverOperation() { + @Override + public void call(RegionServerObserver observer) throws IOException { + observer.postArchiveWALRequest(this); + } + }); + } + public void preReplicateLogEntries() throws IOException { execOperation(coprocEnvironments.isEmpty() ? null : new RegionServerObserverOperation() { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java index a1e4facde712..20c5d0dbef40 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java @@ -57,14 +57,17 @@ import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.PrivateCellUtil; +import org.apache.hadoop.hbase.RegionException; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.exceptions.TimeoutIOException; import org.apache.hadoop.hbase.io.util.MemorySizeUtil; import org.apache.hadoop.hbase.ipc.RpcServer; import org.apache.hadoop.hbase.ipc.ServerCall; import org.apache.hadoop.hbase.log.HBaseMarkers; +import org.apache.hadoop.hbase.regionserver.FlushLifeCycleTracker; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl; +import org.apache.hadoop.hbase.regionserver.RegionServerServices; import org.apache.hadoop.hbase.trace.TraceUtil; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; @@ -718,7 +721,7 @@ private void cleanOldLogs() throws IOException { if (logsToArchive != null) { for (Pair logAndSize : logsToArchive) { this.totalLogSize.addAndGet(-logAndSize.getSecond()); - archiveLogFile(logAndSize.getFirst()); + moveLogFileToArchiveDir(logAndSize.getFirst()); this.walFile2Props.remove(logAndSize.getFirst()); } } @@ -732,7 +735,7 @@ public static Path getWALArchivePath(Path archiveDir, Path p) { return new Path(archiveDir, p.getName()); } - private void archiveLogFile(final Path p) throws IOException { + private void moveLogFileToArchiveDir(final Path p) throws IOException { Path newPath = getWALArchivePath(this.walArchiveDir, p); // Tell our listeners that a log is going to be archived. if (!this.listeners.isEmpty()) { @@ -872,6 +875,33 @@ public Map> rollWriter(boolean force) throws IOException { } } + public void archive(RegionServerServices services) throws IOException{ + if (getNumRolledLogFiles() < 1) { + return; + } + // get the earliest log of this WAL instance + Map.Entry firstWALEntry = this.walFile2Props.firstEntry(); + // flush regions if necessary + Map> regions = + this.sequenceIdAccounting.findLower(firstWALEntry.getValue().encodedName2HighestSequenceId); + if (regions != null) { + for (Map.Entry> entry : regions.entrySet()) { + String encodedRegionName = Bytes.toString(entry.getKey()); + HRegion r = (HRegion) services.getRegion(encodedRegionName); + if (r == null) { + throw new RegionException("Failed to flush of " + encodedRegionName + + " when archive manually, because it is not online on rs"); + } + r.flushcache(entry.getValue(), false, FlushLifeCycleTracker.DUMMY); + } + } + + // move the log file to archive dir + moveLogFileToArchiveDir(firstWALEntry.getKey()); + this.totalLogSize.addAndGet(-firstWALEntry.getValue().logSize); + this.walFile2Props.remove(firstWALEntry.getKey()); + } + // public only until class moves to o.a.h.h.wal /** @return the size of log files in use */ public long getLogFileSize() { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractWALRoller.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractWALRoller.java index a5a0ee3a3225..8af5e24ea9b9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractWALRoller.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractWALRoller.java @@ -30,6 +30,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.regionserver.RegionServerServices; import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL; import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException; import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener; @@ -100,6 +101,14 @@ public void requestRollAll() { } } + public void requestArchive() throws IOException { + synchronized (this) { + for (WAL wal : wals.keySet()) { + wal.archive((RegionServerServices) this.abortable); + } + } + } + protected AbstractWALRoller(String name, Configuration conf, T abortable) { super(name); this.abortable = abortable; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java index dbc08cc84828..9346f1430fed 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.PrivateCellUtil; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl.WriteEntry; +import org.apache.hadoop.hbase.regionserver.RegionServerServices; import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener; import org.apache.hadoop.hbase.regionserver.wal.WALCoprocessorHost; import org.apache.hadoop.hbase.util.CommonFSUtils; @@ -143,6 +144,10 @@ public Map> rollWriter(boolean force) { return rollWriter(); } + @Override + public void archive(RegionServerServices services) { + } + @Override public void shutdown() { if(closed.compareAndSet(false, true)) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java index 902ca6d398a6..0942f51db40a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java @@ -24,6 +24,7 @@ import java.util.Set; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.regionserver.RegionServerServices; import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException; import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener; import org.apache.hadoop.hbase.regionserver.wal.WALCoprocessorHost; @@ -83,6 +84,11 @@ public interface WAL extends Closeable, WALFileLengthProvider { */ Map> rollWriter(boolean force) throws IOException; + /** + * Archive the earliest log file. + */ + void archive(RegionServerServices services) throws IOException; + /** * Stop accepting new writes. If we have unsynced writes still in buffer, sync them. * Extant edits are left in place in backing storage to be replayed later. diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestArchiveWalFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestArchiveWalFromClient.java new file mode 100644 index 000000000000..755a81d916bc --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestArchiveWalFromClient.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.List; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL; +import org.apache.hadoop.hbase.testclassification.ClientTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({MediumTests.class, ClientTests.class}) +public class TestArchiveWalFromClient { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestArchiveWalFromClient.class); + + private static final Logger LOG = LoggerFactory.getLogger(TestArchiveWalFromClient.class); + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final byte[] ROW = Bytes.toBytes("testRow"); + private static final byte[] FAMILY = Bytes.toBytes("testFamily"); + private static final byte[] QUALIFIER = Bytes.toBytes("testQualifier"); + private static final byte[] VALUE = Bytes.toBytes("testValue"); + @Rule + public TestName name = new TestName(); + + public TableName tableName; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniCluster(1); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Before + public void setUp() throws Exception { + tableName = TableName.valueOf(name.getMethodName()); + } + + @After + public void tearDown() throws Exception { + for (TableDescriptor htd : TEST_UTIL.getAdmin().listTableDescriptors()) { + LOG.info("Tear down, remove table=" + htd.getTableName()); + TEST_UTIL.deleteTable(htd.getTableName()); + } + } + + @Test + public void testArchiveEarliestWAL() throws Exception { + try (Table t = TEST_UTIL.createTable(tableName, FAMILY)) { + Put put = new Put(ROW); + put.addColumn(FAMILY, QUALIFIER, VALUE); + t.put(put); + } + try (Admin admin = TEST_UTIL.getAdmin()) { + HRegionServer rs = TEST_UTIL.getHBaseCluster().getRegionServer(0); + assertTrue(getRegionInfo().get(0).getMemStoreDataSize() > 0); + AbstractFSWAL wal = (AbstractFSWAL) rs.getWALs().get(0); + assertEquals(0, wal.getNumRolledLogFiles()); + // do not use admin.rollWALWriter here, since it is async call + wal.rollWriter(); + assertEquals(1, wal.getNumRolledLogFiles()); + admin.archiveWAL(rs.getServerName()); + assertEquals(0, wal.getNumRolledLogFiles()); + // the region should be flushed after archive + assertFalse(getRegionInfo().get(0).getMemStoreDataSize() > 0); + } + } + + private List getRegionInfo() { + return TEST_UTIL.getHBaseCluster().getRegions(tableName); + } + +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java index 1b2d86e19aac..d768473a534e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java @@ -81,6 +81,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ArchiveWALRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ArchiveWALResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearRegionBlockCacheRequest; @@ -515,6 +517,12 @@ public RollWALWriterResponse rollWALWriter(RpcController controller, return null; } + @Override + public ArchiveWALResponse archiveWAL(RpcController controller, + ArchiveWALRequest request) throws ServiceException { + return null; + } + @Override public GetServerInfoResponse getServerInfo(RpcController controller, GetServerInfoRequest request) throws ServiceException { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java index aad0d416d265..9bfa2823b0d6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java @@ -497,6 +497,10 @@ public void rollWALWriter(ServerName serverName) throws IOException, FailedLogCl admin.rollWALWriter(serverName); } + public void archiveWAL(ServerName serverName) throws IOException { + admin.archiveWAL(serverName); + } + public CompactionState getCompactionState(TableName tableName) throws IOException { return admin.getCompactionState(tableName); } diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index f298a123dbcb..e5bbe1b2f01b 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -172,6 +172,12 @@ def wal_roll(server_name) # TODO: remove older hlog_roll version alias hlog_roll wal_roll + #---------------------------------------------------------------------------------------------- + # Requests a regionserver's WAL to archive earliest log file + def wal_archive(server_name) + @admin.archiveWAL(ServerName.valueOf(server_name)) + end + #---------------------------------------------------------------------------------------------- # Requests a table or region split def split(table_or_region_name, split_point = nil) diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb index 1217ca68a59a..fe8e9120c8d6 100644 --- a/hbase-shell/src/main/ruby/shell.rb +++ b/hbase-shell/src/main/ruby/shell.rb @@ -453,6 +453,7 @@ def eval_io(io) unassign zk_dump wal_roll + wal_archive hbck_chore_run catalogjanitor_run catalogjanitor_switch diff --git a/hbase-shell/src/main/ruby/shell/commands/wal_archive.rb b/hbase-shell/src/main/ruby/shell/commands/wal_archive.rb new file mode 100644 index 000000000000..bb70e01ba768 --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/wal_archive.rb @@ -0,0 +1,37 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +module Shell + module Commands + class WalArchive < Command + def help + <<-EOF +Archive the earliest log files. If too many regions need to be flushed, +it will cost a few time. +A 'server_name' is the host, port plus startcode of a regionserver. +For example: host187.example.com,60020,1289493121758 (find servername in +master ui or when you do detailed status in shell) +EOF + end + + def command(server_name) + admin.wal_archive(server_name) + end + end + end +end diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java index 9758d087b709..fedb0b1e2dd3 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java @@ -793,6 +793,11 @@ public void rollWALWriter(ServerName serverName) { } + @Override + public void archiveWAL(ServerName serverName) { + throw new NotImplementedException("archiveWAL not supported in ThriftAdmin"); + } + @Override public CompactionState getCompactionState(TableName tableName) { throw new NotImplementedException("getCompactionState not supported in ThriftAdmin");