Skip to content

Commit

Permalink
HBASE-27688 HFile splitting occurs during bulkload, the CREATE_TIME_T…
Browse files Browse the repository at this point in the history
…S of hfileinfo is 0 (apache#5097)

Co-authored-by: alanzhao <alanzhao@126.com>
Signed-off-by: Duo Zhang <zhangduo@apache.org>
(cherry picked from commit 6920e72)
  • Loading branch information
haohao0103 authored and Apache9 committed Mar 15, 2023
1 parent e2e7968 commit 52d3824
Show file tree
Hide file tree
Showing 2 changed files with 37 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@
import org.apache.hadoop.hbase.security.UserProvider;
import org.apache.hadoop.hbase.security.token.FsDelegationToken;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.FSVisitor;
import org.apache.hadoop.hbase.util.Pair;
Expand Down Expand Up @@ -1170,7 +1171,7 @@ private static void copyHFileHalf(Configuration conf, Path inFile, Path outFile,
.withChecksumType(StoreUtils.getChecksumType(conf))
.withBytesPerCheckSum(StoreUtils.getBytesPerChecksum(conf)).withBlockSize(blocksize)
.withDataBlockEncoding(familyDescriptor.getDataBlockEncoding()).withIncludesTags(true)
.build();
.withCreateTime(EnvironmentEdgeManager.currentTime()).build();
halfWriter = new StoreFileWriter.Builder(conf, cacheConf, fs).withFilePath(outFile)
.withBloomType(bloomFilterType).withFileContext(hFileContext).build();
HFileScanner scanner = halfReader.getScanner(false, false, false);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.tool;

import static org.hamcrest.Matchers.greaterThan;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
Expand Down Expand Up @@ -60,6 +61,7 @@
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CommonFSUtils;
import org.apache.hadoop.hbase.util.HFileTestUtil;
import org.hamcrest.MatcherAssert;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
Expand Down Expand Up @@ -567,6 +569,39 @@ public void testSplitStoreFile() throws IOException {
assertEquals(1000, rowCount);
}

/**
* This method tests that the create_time property of the HFile produced by the splitstorefile
* method is greater than 0 HBASE-27688
*/
@Test
public void testSplitStoreFileWithCreateTimeTS() throws IOException {
Path dir = util.getDataTestDirOnTestFS("testSplitStoreFileWithCreateTimeTS");
FileSystem fs = util.getTestFileSystem();
Path testIn = new Path(dir, "testhfile");
ColumnFamilyDescriptor familyDesc = ColumnFamilyDescriptorBuilder.of(FAMILY);
HFileTestUtil.createHFile(util.getConfiguration(), fs, testIn, FAMILY, QUALIFIER,
Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), 1000);

Path bottomOut = new Path(dir, "bottom.out");
Path topOut = new Path(dir, "top.out");

BulkLoadHFilesTool.splitStoreFile(util.getConfiguration(), testIn, familyDesc,
Bytes.toBytes("ggg"), bottomOut, topOut);

verifyHFileCreateTimeTS(bottomOut);
verifyHFileCreateTimeTS(topOut);
}

private void verifyHFileCreateTimeTS(Path p) throws IOException {
Configuration conf = util.getConfiguration();

try (HFile.Reader reader =
HFile.createReader(p.getFileSystem(conf), p, new CacheConfig(conf), true, conf)) {
long fileCreateTime = reader.getHFileInfo().getHFileContext().getFileCreateTime();
MatcherAssert.assertThat(fileCreateTime, greaterThan(0L));
}
}

@Test
public void testSplitStoreFileWithNoneToNone() throws IOException {
testSplitStoreFileWithDifferentEncoding(DataBlockEncoding.NONE, DataBlockEncoding.NONE);
Expand Down

0 comments on commit 52d3824

Please sign in to comment.