Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[FLASH-270] Delete Range Data At First while Applying Snapshot #75

Merged
merged 35 commits into from
May 30, 2019
Merged
Changes from 1 commit
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
350cd73
change getGCSafePoint in MockPDClient.
solotzg May 20, 2019
65c3037
Add del col to secondary sorting expression.
solotzg May 20, 2019
c92453a
Complete operation in memory.
solotzg May 20, 2019
9fbd896
check all table while applying snapshot.
solotzg May 20, 2019
6e9ded9
!!!tmp
solotzg May 21, 2019
3e34280
Merge remote-tracking branch 'pingcap/master' into FLASH-270
solotzg May 21, 2019
9abf554
Revert "!!!tmp"
solotzg May 21, 2019
93f7d9d
!!!tmp
solotzg May 21, 2019
3e85a92
Small fix.
solotzg May 22, 2019
f3569e6
rollback operation about deleting record.
solotzg May 22, 2019
0b10c65
Optimize executor.
solotzg May 22, 2019
7210284
optimize ReplacingTMTSortedBlockInputStream.
solotzg May 22, 2019
6db5de9
optimize for only need handle, tso, delmark.
solotzg May 23, 2019
acf76b6
import TMTSortCursor.
solotzg May 23, 2019
2a27711
finish.
solotzg May 23, 2019
b0c8399
fix.
solotzg May 23, 2019
7759b63
fix.
solotzg May 23, 2019
1019e44
fix.
solotzg May 23, 2019
146690b
Optimize for count(*).
solotzg May 23, 2019
c860f47
fix bug.
solotzg May 23, 2019
d3a77a5
Fix
solotzg May 24, 2019
dacb64c
optimize TMTSortedBlockInputStream.
solotzg May 24, 2019
3a13a98
Optimize log.
solotzg May 24, 2019
2ef4eaa
no need to report after persisted.
solotzg May 25, 2019
b4d3c12
Optimize memory copy in Region::onCommand.
solotzg May 25, 2019
acea86c
more mock test.
solotzg May 25, 2019
189f1b1
optimize snapshot.
solotzg May 26, 2019
92ee816
Fix Region::compareAndCompleteSnapshot.
solotzg May 26, 2019
e73b8c5
fix compile
solotzg May 26, 2019
2bca1d2
Optimize when pk is int64 or uint64.
solotzg May 27, 2019
68b450d
fix compile.
solotzg May 27, 2019
457c872
Small fix: add const to parameter.
solotzg May 29, 2019
5e0d2c8
optimize performance.
solotzg May 29, 2019
3270582
Format.
solotzg May 29, 2019
f2e51a8
address comment.
solotzg May 30, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Optimize executor.
solotzg committed May 22, 2019
commit 0b10c65d996ddd3e52aa991c48796f1d6041a448
59 changes: 29 additions & 30 deletions dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
Original file line number Diff line number Diff line change
@@ -217,9 +217,13 @@ BlockInputStreams MergeTreeDataSelectExecutor::read(const Names & column_names_t
}
}

ASTSelectQuery & select = typeid_cast<ASTSelectQuery &>(*query_info.query);

bool pk_is_uint64 = false;

if (is_txn_engine)
if (!is_txn_engine)
;
else if (!select.no_kvstore)
{
handle_col_name = data.getPrimarySortDescription()[0].column_name;

@@ -228,40 +232,33 @@ BlockInputStreams MergeTreeDataSelectExecutor::read(const Names & column_names_t
if (std::strcmp(pk_type, TypeName<UInt64>::get()) == 0)
pk_is_uint64 = true;

ASTSelectQuery & select = typeid_cast<ASTSelectQuery &>(*query_info.query);

TMTContext & tmt = context.getTMTContext();

if (!tmt.isInitialized())
throw Exception("TMTContext is not initialized", ErrorCodes::LOGICAL_ERROR);

if (select.no_kvstore)
regions_query_info.clear();
else
{
for (const auto & query_info : regions_query_info)
kvstore_region.emplace(query_info.region_id, tmt.getKVStore()->getRegion(query_info.region_id));
for (const auto & query_info : regions_query_info)
kvstore_region.emplace(query_info.region_id, tmt.getKVStore()->getRegion(query_info.region_id));

if (regions_query_info.empty())
{
tmt.getRegionTableMut().traverseRegionsByTable(data.table_info->id, [&](std::vector<std::pair<RegionID, RegionPtr>> & regions) {
for (const auto & [id, region] : regions)
{
kvstore_region.emplace(id, region);
if (region == nullptr)
// maybe region is removed.
regions_query_info.push_back({id, InvalidRegionVersion, InvalidRegionVersion, {0, 0}});
else
regions_query_info.push_back(
{id, region->version(), region->confVer(), region->getHandleRangeByTable(data.table_info->id)});
}
});
}

if (kvstore_region.size() != regions_query_info.size())
throw Exception("Duplicate region id", ErrorCodes::LOGICAL_ERROR);
if (regions_query_info.empty())
{
tmt.getRegionTableMut().traverseRegionsByTable(data.table_info->id, [&](std::vector<std::pair<RegionID, RegionPtr>> & regions) {
for (const auto & [id, region] : regions)
{
kvstore_region.emplace(id, region);
if (region == nullptr)
// maybe region is removed.
regions_query_info.push_back({id, InvalidRegionVersion, InvalidRegionVersion, {0, 0}});
else
regions_query_info.push_back(
{id, region->version(), region->confVer(), region->getHandleRangeByTable(data.table_info->id)});
}
});
}

if (kvstore_region.size() != regions_query_info.size())
throw Exception("Duplicate region id", ErrorCodes::LOGICAL_ERROR);

std::sort(regions_query_info.begin(), regions_query_info.end());

{ // check all regions
@@ -483,8 +480,6 @@ BlockInputStreams MergeTreeDataSelectExecutor::read(const Names & column_names_t
RelativeSize relative_sample_size = 0;
RelativeSize relative_sample_offset = 0;

ASTSelectQuery & select = typeid_cast<ASTSelectQuery &>(*query_info.query);

auto select_sample_size = select.sample_size();
auto select_sample_offset = select.sample_offset();

@@ -914,7 +909,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::read(const Names & column_names_t
res.emplace_back(std::make_shared<BlocksListBlockInputStream>(std::move(blocks)));
}
}
else
else if (!select.no_kvstore)
{
const auto func_make_merge_tree_input = [&](const RangesInDataPart & part, const MarkRanges & mark_ranges) {
return std::make_shared<MergeTreeBlockInputStream>(data, part.data_part, max_block_size,
@@ -1074,6 +1069,10 @@ BlockInputStreams MergeTreeDataSelectExecutor::read(const Names & column_names_t
res.emplace_back(std::make_shared<UnionBlockInputStream<>>(union_regions_stream, nullptr, 1));
}
}
else
{
// no use for select nokvstore ...
}
}
else if (data.merging_params.mode == MergeTreeData::MergingParams::Mutable && !select.raw_for_mutable)
{