Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

optimize ps v3 restore #5163

Merged
merged 8 commits into from
Jun 22, 2022
Merged

optimize ps v3 restore #5163

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
943b0e9
optimize ps v3 restore
hehechen Jun 16, 2022
5447f4f
Update dbms/src/Storages/Page/V3/PageDirectory.h
hehechen Jun 20, 2022
2bac4eb
Update dbms/src/Storages/Page/V3/PageDirectoryFactory.cpp
hehechen Jun 20, 2022
ca66832
fix comments
hehechen Jun 20, 2022
396420c
fix
hehechen Jun 20, 2022
b9c7c3f
Merge branch 'master' into v3restore
JaySon-Huang Jun 21, 2022
818e7df
Merge branch 'master' into v3restore
ti-chi-bot Jun 22, 2022
2fa35e1
Merge branch 'master' into v3restore
ti-chi-bot Jun 22, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 17 additions & 8 deletions dbms/src/Storages/Page/V3/PageDirectory.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -479,7 +479,8 @@ bool VersionedPageEntries::cleanOutdatedEntries(
UInt64 lowest_seq,
std::map<PageIdV3Internal, std::pair<PageVersion, Int64>> * normal_entries_to_deref,
PageEntriesV3 & entries_removed,
const PageLock & /*page_lock*/)
const PageLock & /*page_lock*/,
bool need_fill_entries_removed)
{
if (type == EditRecordType::VAR_EXTERNAL)
{
Expand Down Expand Up @@ -541,7 +542,10 @@ bool VersionedPageEntries::cleanOutdatedEntries(
{
if (iter->second.being_ref_count == 1)
{
entries_removed.emplace_back(iter->second.entry);
if (need_fill_entries_removed)
{
entries_removed.emplace_back(iter->second.entry);
}
iter = entries.erase(iter);
}
// The `being_ref_count` for this version is valid. While for older versions,
Expand All @@ -551,7 +555,10 @@ bool VersionedPageEntries::cleanOutdatedEntries(
else
{
// else there are newer "entry" in the version list, the outdated entries should be removed
entries_removed.emplace_back(iter->second.entry);
if (need_fill_entries_removed)
{
entries_removed.emplace_back(iter->second.entry);
}
iter = entries.erase(iter);
}
}
Expand All @@ -564,7 +571,7 @@ bool VersionedPageEntries::cleanOutdatedEntries(
return entries.empty() || (entries.size() == 1 && entries.begin()->second.isDelete());
}

bool VersionedPageEntries::derefAndClean(UInt64 lowest_seq, PageIdV3Internal page_id, const PageVersion & deref_ver, const Int64 deref_count, PageEntriesV3 & entries_removed)
bool VersionedPageEntries::derefAndClean(UInt64 lowest_seq, PageIdV3Internal page_id, const PageVersion & deref_ver, const Int64 deref_count, PageEntriesV3 & entries_removed, bool need_fill_entries_removed)
{
auto page_lock = acquireLock();
if (type == EditRecordType::VAR_EXTERNAL)
Expand Down Expand Up @@ -601,7 +608,7 @@ bool VersionedPageEntries::derefAndClean(UInt64 lowest_seq, PageIdV3Internal pag

// Clean outdated entries after decreased the ref-counter
// set `normal_entries_to_deref` to be nullptr to ignore cleaning ref-var-entries
return cleanOutdatedEntries(lowest_seq, /*normal_entries_to_deref*/ nullptr, entries_removed, page_lock);
return cleanOutdatedEntries(lowest_seq, /*normal_entries_to_deref*/ nullptr, entries_removed, page_lock, need_fill_entries_removed);
}

throw Exception(fmt::format("calling derefAndClean with invalid state [state={}]", toDebugString()));
Expand Down Expand Up @@ -1239,7 +1246,7 @@ bool PageDirectory::tryDumpSnapshot(const ReadLimiterPtr & read_limiter, const W
return done_any_io;
}

PageEntriesV3 PageDirectory::gcInMemEntries()
PageEntriesV3 PageDirectory::gcInMemEntries(bool need_remove_blob)
{
UInt64 lowest_seq = sequence.load();

Expand Down Expand Up @@ -1304,7 +1311,8 @@ PageEntriesV3 PageDirectory::gcInMemEntries()
lowest_seq,
&normal_entries_to_deref,
all_del_entries,
iter->second->acquireLock());
iter->second->acquireLock(),
need_remove_blob);

{
std::unique_lock write_lock(table_rw_mutex);
Expand Down Expand Up @@ -1342,7 +1350,8 @@ PageEntriesV3 PageDirectory::gcInMemEntries()
page_id,
/*deref_ver=*/deref_counter.first,
/*deref_count=*/deref_counter.second,
all_del_entries);
all_del_entries,
need_remove_blob);

if (all_deleted)
{
Expand Down
8 changes: 5 additions & 3 deletions dbms/src/Storages/Page/V3/PageDirectory.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,13 +224,15 @@ class VersionedPageEntries
UInt64 lowest_seq,
std::map<PageIdV3Internal, std::pair<PageVersion, Int64>> * normal_entries_to_deref,
PageEntriesV3 & entries_removed,
const PageLock & page_lock);
const PageLock & page_lock,
bool need_fill_entries_removed);
bool derefAndClean(
UInt64 lowest_seq,
PageIdV3Internal page_id,
const PageVersion & deref_ver,
Int64 deref_count,
PageEntriesV3 & entries_removed);
PageEntriesV3 & entries_removed,
bool need_fill_entries_removed);
hehechen marked this conversation as resolved.
Show resolved Hide resolved

void collapseTo(UInt64 seq, PageIdV3Internal page_id, PageEntriesEdit & edit);

Expand Down Expand Up @@ -360,7 +362,7 @@ class PageDirectory

bool tryDumpSnapshot(const ReadLimiterPtr & read_limiter = nullptr, const WriteLimiterPtr & write_limiter = nullptr);

PageEntriesV3 gcInMemEntries();
PageEntriesV3 gcInMemEntries(bool need_remove_blob = true);
hehechen marked this conversation as resolved.
Show resolved Hide resolved

std::set<PageId> getAliveExternalIds(NamespaceId ns_id) const;

Expand Down
2 changes: 1 addition & 1 deletion dbms/src/Storages/Page/V3/PageDirectoryFactory.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ PageDirectoryPtr PageDirectoryFactory::createFromReader(String storage_name, WAL

// After restoring from the disk, we need cleanup all invalid entries in memory, or it will
// try to run GC again on some entries that are already marked as invalid in BlobStore.
dir->gcInMemEntries();
dir->gcInMemEntries(false);
hehechen marked this conversation as resolved.
Show resolved Hide resolved
LOG_FMT_INFO(DB::Logger::get("PageDirectoryFactory", storage_name), "PageDirectory restored [max_page_id={}] [max_applied_ver={}]", dir->getMaxId(), dir->sequence);

if (blob_stats)
Expand Down