Skip to content

Commit

Permalink
ColumnFamilySet
Browse files Browse the repository at this point in the history
Summary:
I created a separate class ColumnFamilySet to keep track of column families. Before we did this in VersionSet and I believe this approach is cleaner.

Let me know if you have any comments. I will commit tomorrow.

Test Plan: make check

Reviewers: dhruba, haobo, kailiu, sdong

CC: leveldb

Differential Revision: https://reviews.facebook.net/D15357
  • Loading branch information
igorcanadi committed Jan 23, 2014
1 parent f9a25dd commit 7c5e583
Show file tree
Hide file tree
Showing 10 changed files with 410 additions and 192 deletions.
86 changes: 86 additions & 0 deletions db/column_family.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
#include "db/column_family.h"
#include "db/version_set.h"

namespace rocksdb {

ColumnFamilyData::ColumnFamilyData(uint32_t id, const std::string& name,
Version* dummy_versions,
const ColumnFamilyOptions& options)
: id(id),
name(name),
dummy_versions(dummy_versions),
current(nullptr),
options(options) {}

ColumnFamilyData::~ColumnFamilyData() {
// List must be empty
assert(dummy_versions->next_ == dummy_versions);
delete dummy_versions;
}

ColumnFamilySet::ColumnFamilySet() : max_column_family_(0) {}

ColumnFamilySet::~ColumnFamilySet() {
for (auto& cfd : column_family_data_) {
delete cfd.second;
}
for (auto& cfd : droppped_column_families_) {
delete cfd;
}
}

ColumnFamilyData* ColumnFamilySet::GetDefault() const {
auto ret = GetColumnFamily(0);
assert(ret != nullptr); // default column family should always exist
return ret;
}

ColumnFamilyData* ColumnFamilySet::GetColumnFamily(uint32_t id) const {
auto cfd_iter = column_family_data_.find(id);
if (cfd_iter != column_family_data_.end()) {
return cfd_iter->second;
} else {
return nullptr;
}
}

bool ColumnFamilySet::Exists(uint32_t id) {
return column_family_data_.find(id) != column_family_data_.end();
}

bool ColumnFamilySet::Exists(const std::string& name) {
return column_families_.find(name) != column_families_.end();
}

uint32_t ColumnFamilySet::GetID(const std::string& name) {
auto cfd_iter = column_families_.find(name);
assert(cfd_iter != column_families_.end());
return cfd_iter->second;
}

uint32_t ColumnFamilySet::GetNextColumnFamilyID() {
return ++max_column_family_;
}

ColumnFamilyData* ColumnFamilySet::CreateColumnFamily(
const std::string& name, uint32_t id, Version* dummy_versions,
const ColumnFamilyOptions& options) {
assert(column_families_.find(name) == column_families_.end());
column_families_.insert({name, id});
ColumnFamilyData* new_cfd =
new ColumnFamilyData(id, name, dummy_versions, options);
column_family_data_.insert({id, new_cfd});
max_column_family_ = std::max(max_column_family_, id);
return new_cfd;
}

void ColumnFamilySet::DropColumnFamily(uint32_t id) {
auto cfd = column_family_data_.find(id);
assert(cfd != column_family_data_.end());
column_families_.erase(cfd->second->name);
cfd->second->current->Unref();
droppped_column_families_.push_back(cfd->second);
column_family_data_.erase(cfd);
}

} // namespace rocksdb
87 changes: 87 additions & 0 deletions db/column_family.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.

#pragma once

#include "rocksdb/options.h"

#include <map>
#include <string>
#include <vector>

namespace rocksdb {

class Version;
class VersionSet;

// column family metadata
struct ColumnFamilyData {
uint32_t id;
std::string name;
Version* dummy_versions; // Head of circular doubly-linked list of versions.
Version* current; // == dummy_versions->prev_
ColumnFamilyOptions options;

ColumnFamilyData(uint32_t id, const std::string& name,
Version* dummy_versions, const ColumnFamilyOptions& options);
~ColumnFamilyData();
};

class ColumnFamilySet {
public:
class iterator {
public:
explicit iterator(
std::unordered_map<uint32_t, ColumnFamilyData*>::iterator itr)
: itr_(itr) {}
iterator& operator++() {
++itr_;
return *this;
}
bool operator!=(const iterator& other) { return this->itr_ != other.itr_; }
ColumnFamilyData* operator*() { return itr_->second; }
private:
std::unordered_map<uint32_t, ColumnFamilyData*>::iterator itr_;
};

ColumnFamilySet();
~ColumnFamilySet();

ColumnFamilyData* GetDefault() const;
// GetColumnFamily() calls return nullptr if column family is not found
ColumnFamilyData* GetColumnFamily(uint32_t id) const;
bool Exists(uint32_t id);
bool Exists(const std::string& name);
uint32_t GetID(const std::string& name);
// this call will return the next available column family ID. it guarantees
// that there is no column family with id greater than or equal to the
// returned value in the current running instance. It does not, however,
// guarantee that the returned ID is unique accross RocksDB restarts.
// For example, if a client adds a column family 6 and then drops it,
// after a restart, we might reuse column family 6 ID.
uint32_t GetNextColumnFamilyID();

ColumnFamilyData* CreateColumnFamily(const std::string& name, uint32_t id,
Version* dummy_version,
const ColumnFamilyOptions& options);
void DropColumnFamily(uint32_t id);

iterator begin() { return iterator(column_family_data_.begin()); }
iterator end() { return iterator(column_family_data_.end()); }

private:
std::unordered_map<std::string, uint32_t> column_families_;
std::unordered_map<uint32_t, ColumnFamilyData*> column_family_data_;
// we need to keep them alive because we still can't control the lifetime of
// all of column family data members (options for example)
std::vector<ColumnFamilyData*> droppped_column_families_;
uint32_t max_column_family_;
};

} // namespace rocksdb
2 changes: 1 addition & 1 deletion db/column_family_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ TEST(ColumnFamilyTest, AddDrop) {
Close();

vector<string> families;
DB::ListColumnFamilies(db_options_, dbname_, &families);
ASSERT_OK(DB::ListColumnFamilies(db_options_, dbname_, &families));
sort(families.begin(), families.end());
ASSERT_TRUE(families == vector<string>({"default", "four", "one", "three"}));
}
Expand Down
64 changes: 19 additions & 45 deletions db/db_impl.cc
Original file line number Diff line number Diff line change
Expand Up @@ -913,21 +913,8 @@ Status DBImpl::Recover(
}
}

Status s = versions_->Recover();
Status s = versions_->Recover(column_families);
if (s.ok()) {
if (column_families.size() != versions_->column_families_.size()) {
return Status::InvalidArgument("Column family specifications mismatch");
}
for (auto cf : column_families) {
auto cf_iter = versions_->column_families_.find(cf.name);
if (cf_iter == versions_->column_families_.end()) {
return Status::InvalidArgument("Column family specifications mismatch");
}
auto cf_data_iter = versions_->column_family_data_.find(cf_iter->second);
assert(cf_data_iter != versions_->column_family_data_.end());
cf_data_iter->second->options = cf.options;
}

SequenceNumber max_sequence(0);

// Recover from all newer log files than the ones named in the
Expand Down Expand Up @@ -2933,11 +2920,13 @@ std::vector<Status> DBImpl::MultiGet(
Status DBImpl::CreateColumnFamily(const ColumnFamilyOptions& options,
const std::string& column_family_name,
ColumnFamilyHandle* handle) {
if (!versions_->GetColumnFamilySet()->Exists(column_family_name)) {
return Status::InvalidArgument("Column family already exists");
}
VersionEdit edit;
edit.AddColumnFamily(column_family_name);
MutexLock l(&mutex_);
++versions_->max_column_family_;
handle->id = versions_->max_column_family_;
handle->id = versions_->GetColumnFamilySet()->GetNextColumnFamilyID();
edit.SetColumnFamily(handle->id);
Status s = versions_->LogAndApply(&edit, &mutex_);
if (s.ok()) {
Expand All @@ -2948,21 +2937,16 @@ Status DBImpl::CreateColumnFamily(const ColumnFamilyOptions& options,
}

Status DBImpl::DropColumnFamily(const ColumnFamilyHandle& column_family) {
// TODO this is not good. implement some sort of refcounting
// column family data and only delete when refcount goes to 0
// We don't want to delete column family if there is a compaction going on,
// or if there are some outstanding iterators
if (column_family.id == 0) {
return Status::InvalidArgument("Can't drop default column family");
}
VersionEdit edit;
edit.DropColumnFamily();
edit.SetColumnFamily(column_family.id);
MutexLock l(&mutex_);
auto data_iter = versions_->column_family_data_.find(column_family.id);
if (data_iter == versions_->column_family_data_.end()) {
if (!versions_->GetColumnFamilySet()->Exists(column_family.id)) {
return Status::NotFound("Column family not found");
}
VersionEdit edit;
edit.DropColumnFamily();
edit.SetColumnFamily(column_family.id);
Status s = versions_->LogAndApply(&edit, &mutex_);
if (s.ok()) {
// remove from internal data structures
Expand Down Expand Up @@ -3968,10 +3952,16 @@ Status DB::OpenWithColumnFamilies(
// set column family handles
handles->clear();
for (auto cf : column_families) {
auto cf_iter = impl->versions_->column_families_.find(cf.name);
assert(cf_iter != impl->versions_->column_families_.end());
handles->push_back(ColumnFamilyHandle(cf_iter->second));
if (!impl->versions_->GetColumnFamilySet()->Exists(cf.name)) {
s = Status::InvalidArgument("Column family not found: ", cf.name);
handles->clear();
break;
}
uint32_t id = impl->versions_->GetColumnFamilySet()->GetID(cf.name);
handles->push_back(ColumnFamilyHandle(id));
}
}
if (s.ok()) {
delete impl->InstallSuperVersion(new DBImpl::SuperVersion());
impl->mem_->SetLogNumber(impl->logfile_number_);
impl->DeleteObsoleteFiles();
Expand Down Expand Up @@ -4006,23 +3996,7 @@ Status DB::OpenWithColumnFamilies(
Status DB::ListColumnFamilies(const DBOptions& db_options,
const std::string& name,
std::vector<std::string>* column_families) {
Options options(db_options, ColumnFamilyOptions());
InternalKeyComparator* icmp = new InternalKeyComparator(options.comparator);
TableCache* table_cache = new TableCache(name, &options, EnvOptions(options),
db_options.max_open_files - 10);
VersionSet* version_set =
new VersionSet(name, &options, EnvOptions(options), table_cache, icmp);

version_set->Recover();
column_families->clear();
for (auto cf : version_set->column_families_) {
column_families->push_back(cf.first);
}

delete version_set;
delete table_cache;
delete icmp;
return Status::NotSupported("Working on it");
return VersionSet::ListColumnFamilies(column_families, name, db_options.env);
}

Snapshot::~Snapshot() {
Expand Down
4 changes: 3 additions & 1 deletion db/db_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -5035,7 +5035,9 @@ void BM_LogAndApply(int iters, int num_base_files) {
Options options;
EnvOptions sopt;
VersionSet vset(dbname, &options, sopt, nullptr, &cmp);
ASSERT_OK(vset.Recover());
std::vector<ColumnFamilyDescriptor> dummy;
dummy.push_back(ColumnFamilyDescriptor());
ASSERT_OK(vset.Recover(dummy));
VersionEdit vbase;
uint64_t fnum = 1;
for (int i = 0; i < num_base_files; i++) {
Expand Down
Loading

0 comments on commit 7c5e583

Please sign in to comment.