From 696a60f1ab18ca8a0cc4ea324356569d19825579 Mon Sep 17 00:00:00 2001 From: chenfu Date: Sun, 14 Oct 2018 13:46:42 +0800 Subject: [PATCH 01/13] support DBWithTTL Open Interface --- librocksdb_sys/crocksdb/c.cc | 53 ++++++++++++++++++++++++++++ librocksdb_sys/crocksdb/crocksdb/c.h | 10 ++++++ librocksdb_sys/src/lib.rs | 17 +++++++++ src/rocksdb.rs | 52 +++++++++++++++++++++++---- 4 files changed, 125 insertions(+), 7 deletions(-) diff --git a/librocksdb_sys/crocksdb/c.cc b/librocksdb_sys/crocksdb/c.cc index e12107829..75471c4de 100644 --- a/librocksdb_sys/crocksdb/c.cc +++ b/librocksdb_sys/crocksdb/c.cc @@ -32,8 +32,10 @@ #include "rocksdb/utilities/backupable_db.h" #include "rocksdb/utilities/debug.h" #include "rocksdb/utilities/options_util.h" +#include "rocksdb/utilities/db_ttl.h" #include "rocksdb/write_batch.h" + #include "db/column_family.h" #include "table/sst_file_writer_collectors.h" #include "table/table_reader.h" @@ -64,6 +66,7 @@ using rocksdb::Comparator; using rocksdb::CompressionType; using rocksdb::WALRecoveryMode; using rocksdb::DB; +using rocksdb::DBWithTTL; using rocksdb::DBOptions; using rocksdb::Env; using rocksdb::EnvOptions; @@ -522,6 +525,20 @@ crocksdb_t* crocksdb_open( return result; } +crocksdb_t* crocksdb_open_with_ttl( + const crocksdb_options_t* options, + const char* name, + int ttl, + char** errptr) { + DBWithTTL* db; + if (SaveError(errptr, DBWithTTL::Open(options->rep, std::string(name), &db, ttl))) { + return nullptr; + } + crocksdb_t* result = new crocksdb_t; + result->rep = db; + return result; +} + crocksdb_t* crocksdb_open_for_read_only( const crocksdb_options_t* options, const char* name, @@ -666,6 +683,42 @@ crocksdb_t* crocksdb_open_column_families( return result; } +crocksdb_t* crocksdb_open_column_families_with_ttl( + const crocksdb_options_t* db_options, + const char* name, + int num_column_families, + const char** column_family_names, + const crocksdb_options_t** column_family_options, + crocksdb_column_family_handle_t** column_family_handles, + const int32_t* ttl_array, + bool read_only, + char** errptr) { + std::vector column_families; + std::vector ttls; + for (int i = 0; i < num_column_families; i++) { + column_families.push_back(ColumnFamilyDescriptor( + std::string(column_family_names[i]), + ColumnFamilyOptions(column_family_options[i]->rep))); + ttls.push_back(ttl_array[i]); + } + + DBWithTTL* db; + std::vector handles; + if (SaveError(errptr, DBWithTTL::Open(DBOptions(db_options->rep), + std::string(name), column_families, &handles, &db, ttls, read_only))) { + return nullptr; + } + + for (size_t i = 0; i < handles.size(); i++) { + crocksdb_column_family_handle_t* c_handle = new crocksdb_column_family_handle_t; + c_handle->rep = handles[i]; + column_family_handles[i] = c_handle; + } + crocksdb_t* result = new crocksdb_t; + result->rep = db; + return result; +} + crocksdb_t* crocksdb_open_for_read_only_column_families( const crocksdb_options_t* db_options, const char* name, diff --git a/librocksdb_sys/crocksdb/crocksdb/c.h b/librocksdb_sys/crocksdb/crocksdb/c.h index f8b65d1c2..208c31268 100644 --- a/librocksdb_sys/crocksdb/crocksdb/c.h +++ b/librocksdb_sys/crocksdb/crocksdb/c.h @@ -165,6 +165,9 @@ typedef enum crocksdb_table_property_t { extern C_ROCKSDB_LIBRARY_API crocksdb_t* crocksdb_open( const crocksdb_options_t* options, const char* name, char** errptr); +extern C_ROCKSDB_LIBRARY_API crocksdb_t* crocksdb_open_with_ttl( + const crocksdb_options_t* options, const char* name, int ttl, char** errptr); + extern C_ROCKSDB_LIBRARY_API crocksdb_t* crocksdb_open_for_read_only( const crocksdb_options_t* options, const char* name, unsigned char error_if_log_file_exist, char** errptr); @@ -223,6 +226,13 @@ extern C_ROCKSDB_LIBRARY_API crocksdb_t* crocksdb_open_column_families( const crocksdb_options_t** column_family_options, crocksdb_column_family_handle_t** column_family_handles, char** errptr); +extern C_ROCKSDB_LIBRARY_API crocksdb_t* crocksdb_open_column_families_with_ttl( + const crocksdb_options_t* options, const char* name, int num_column_families, + const char** column_family_names, + const crocksdb_options_t** column_family_options, + crocksdb_column_family_handle_t** column_family_handles, + const int32_t* ttl_array, bool read_only, char** errptr); + extern C_ROCKSDB_LIBRARY_API crocksdb_t* crocksdb_open_for_read_only_column_families( const crocksdb_options_t* options, const char* name, int num_column_families, diff --git a/librocksdb_sys/src/lib.rs b/librocksdb_sys/src/lib.rs index 1bf905ae2..1b53f03df 100644 --- a/librocksdb_sys/src/lib.rs +++ b/librocksdb_sys/src/lib.rs @@ -517,6 +517,12 @@ extern "C" { path: *const c_char, err: *mut *mut c_char, ) -> *mut DBInstance; + pub fn crocksdb_open_with_ttl( + options: *mut Options, + path: *const c_char, + ttl: c_int, + err: *mut *mut c_char, + ) -> *mut DBInstance; pub fn crocksdb_open_for_read_only( options: *mut Options, path: *const c_char, @@ -852,6 +858,17 @@ extern "C" { column_family_handles: *const *mut DBCFHandle, err: *mut *mut c_char, ) -> *mut DBInstance; + pub fn crocksdb_open_column_families_with_ttl( + options: *const Options, + path: *const c_char, + num_column_families: c_int, + column_family_names: *const *const c_char, + column_family_options: *const *const Options, + column_family_handles: *const *mut DBCFHandle, + ttl_array: *const c_int, + read_only: bool, + err: *mut *mut c_char, + ) -> *mut DBInstance; pub fn crocksdb_open_for_read_only_column_families( options: *const Options, path: *const c_char, diff --git a/src/rocksdb.rs b/src/rocksdb.rs index e57ee1fcf..25baf0d5f 100644 --- a/src/rocksdb.rs +++ b/src/rocksdb.rs @@ -370,13 +370,27 @@ impl DB { DB::open_cf(opts, path, cfds) } + pub fn open_with_ttl(opts: DBOptions, path: &str, ttls: &[i32]) -> Result { + let cfds: Vec<&str> = vec![]; + if ttls.len() > 0 { + DB::open_cf_with_ttl(opts, path, cfds, ttls) + } else { + DB::open_cf(opts, path, cfds) + } + } + pub fn open_cf<'a, T>(opts: DBOptions, path: &str, cfds: Vec) -> Result where T: Into>, { - DB::open_cf_internal(opts, path, cfds, None) + DB::open_cf_internal(opts, path, cfds, &[], None) } + pub fn open_cf_with_ttl<'a, T>(opts: DBOptions, path: &str, cfds: Vec, ttls: &[i32]) ->Result + where + T: Into>, { DB::open_cf_internal(opts, path, cfds, ttls,None) } + + pub fn open_for_read_only( opts: DBOptions, path: &str, @@ -395,13 +409,14 @@ impl DB { where T: Into>, { - DB::open_cf_internal(opts, path, cfds, Some(error_if_log_file_exist)) + DB::open_cf_internal(opts, path, cfds, &[], Some(error_if_log_file_exist)) } fn open_cf_internal<'a, T>( opts: DBOptions, path: &str, cfds: Vec, + ttls: &[i32], // if none, open for read write mode. // otherwise, open for read only. error_if_log_file_exist: Option, @@ -441,6 +456,11 @@ impl DB { } else { false }; + let with_ttl = if ttls.len() > 0 && ttls.len() == cf_names.len() { + true + } else { + false + }; let db = { let db_options = opts.inner; let db_path = cpath.as_ptr(); @@ -448,9 +468,11 @@ impl DB { let db_cf_ptrs = cf_names.as_ptr(); let db_cf_opts = cf_options.as_ptr(); let db_cf_handles = cf_handles.as_ptr(); - if let Some(flag) = error_if_log_file_exist { - unsafe { - ffi_try!(crocksdb_open_for_read_only_column_families( + + if !with_ttl { + if let Some(flag) = error_if_log_file_exist { + unsafe { + ffi_try!(crocksdb_open_for_read_only_column_families( db_options, db_path, db_cfs_count, @@ -459,19 +481,35 @@ impl DB { db_cf_handles, flag )) + } + } else { + unsafe { + ffi_try!(crocksdb_open_column_families( + db_options, + db_path, + db_cfs_count, + db_cf_ptrs, + db_cf_opts, + db_cf_handles + )) + } } } else { + let ttl_array = ttls.as_ptr() as *const c_int; unsafe { - ffi_try!(crocksdb_open_column_families( + ffi_try!(crocksdb_open_column_families_with_ttl( db_options, db_path, db_cfs_count, db_cf_ptrs, db_cf_opts, - db_cf_handles + db_cf_handles, + ttl_array, + readonly )) } } + }; if cf_handles.iter().any(|h| h.is_null()) { return Err(ERR_NULL_CF_HANDLE.to_owned()); From 127aa1af92e79cd2b9896cabda31b705351fc98b Mon Sep 17 00:00:00 2001 From: chenfu Date: Tue, 16 Oct 2018 10:31:04 +0800 Subject: [PATCH 02/13] add DBWithTTL_Open test --- tests/test.rs | 1 + tests/test_ttl.rs | 59 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 tests/test_ttl.rs diff --git a/tests/test.rs b/tests/test.rs index cdf88a176..dc9fad552 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -21,3 +21,4 @@ mod test_rocksdb_options; mod test_slice_transform; mod test_statistics; mod test_table_properties; +mod test_ttl; diff --git a/tests/test_ttl.rs b/tests/test_ttl.rs new file mode 100644 index 000000000..f5afe178b --- /dev/null +++ b/tests/test_ttl.rs @@ -0,0 +1,59 @@ +use rocksdb::{ColumnFamilyOptions, DBOptions, Writable, DB}; +use tempdir::TempDir; + +#[test] +pub fn test_ttl() { + let path = TempDir::new("_rust_rocksdb_ttl_test").expect(""); + let path_str = path.path().to_str().unwrap(); + + // should be able to open db with ttl + { + let mut opts = DBOptions::new(); + let cf_opts = ColumnFamilyOptions::new(); + let ttl = 10; + opts.create_if_missing(true); + // cf_opts.set_num_levels(2); + // assert_eq!(2, cf_opts.get_num_levels()); + let mut db = match DB::open_cf_with_ttl( + opts, + path.path().to_str().unwrap(), + vec![("default", cf_opts)], + &[ttl] + ) { + Ok(db) => { + println!("successfully opened db with ttl"); + db + } + Err(e) => panic!("failed to open db with ttl: {}", e), + }; + + match db.create_cf("cf1") { + Ok(_) => println!("cf1 created successfully"), + Err(e) => { + panic!("could not create column family: {}", e); + } + } + assert_eq!(db.cf_names(), vec!["cf1", "default"]); + drop(db); + } + + + // should be able to write, read over a cf with ttl + { + let cf_opts = ColumnFamilyOptions::new(); + let ttl = 0; + let db = match DB::open_cf_with_ttl(DBOptions::new(), path_str, vec![("cf1", cf_opts)], &[ttl]) { + Ok(db) => { + println!("successfully opened cf with ttl"); + db + } + Err(e) => panic!("failed to open cf with ttl: {}", e), + }; + let cf1 = db.cf_handle("cf1").unwrap(); + assert!(db.put_cf(cf1, b"k1", b"v1").is_ok()); + assert!(db.get_cf(cf1, b"k1").unwrap().unwrap().to_utf8().unwrap() == "v1"); + let p = db.put_cf(cf1, b"k1", b"a"); + assert!(p.is_ok()); + } + +} From ca382f44a9deb65118e2d824c10adbabf27ee624 Mon Sep 17 00:00:00 2001 From: chenfu Date: Wed, 17 Oct 2018 19:00:53 +0800 Subject: [PATCH 03/13] run cargo fmt --all --- src/rocksdb.rs | 47 +++++++++++++++++++++++++++-------------------- tests/test_ttl.rs | 14 ++++++++------ 2 files changed, 35 insertions(+), 26 deletions(-) diff --git a/src/rocksdb.rs b/src/rocksdb.rs index 25baf0d5f..0576bd3af 100644 --- a/src/rocksdb.rs +++ b/src/rocksdb.rs @@ -386,10 +386,17 @@ impl DB { DB::open_cf_internal(opts, path, cfds, &[], None) } - pub fn open_cf_with_ttl<'a, T>(opts: DBOptions, path: &str, cfds: Vec, ttls: &[i32]) ->Result + pub fn open_cf_with_ttl<'a, T>( + opts: DBOptions, + path: &str, + cfds: Vec, + ttls: &[i32], + ) -> Result where - T: Into>, { DB::open_cf_internal(opts, path, cfds, ttls,None) } - + T: Into>, + { + DB::open_cf_internal(opts, path, cfds, ttls, None) + } pub fn open_for_read_only( opts: DBOptions, @@ -473,25 +480,25 @@ impl DB { if let Some(flag) = error_if_log_file_exist { unsafe { ffi_try!(crocksdb_open_for_read_only_column_families( - db_options, - db_path, - db_cfs_count, - db_cf_ptrs, - db_cf_opts, - db_cf_handles, - flag - )) + db_options, + db_path, + db_cfs_count, + db_cf_ptrs, + db_cf_opts, + db_cf_handles, + flag + )) } } else { unsafe { ffi_try!(crocksdb_open_column_families( - db_options, - db_path, - db_cfs_count, - db_cf_ptrs, - db_cf_opts, - db_cf_handles - )) + db_options, + db_path, + db_cfs_count, + db_cf_ptrs, + db_cf_opts, + db_cf_handles + )) } } } else { @@ -509,7 +516,6 @@ impl DB { )) } } - }; if cf_handles.iter().any(|h| h.is_null()) { return Err(ERR_NULL_CF_HANDLE.to_owned()); @@ -2492,7 +2498,8 @@ mod test { db1.put(b"k2", b"v2").unwrap(); db1.flush(true).unwrap(); db1.compact_range(None, None); - }).unwrap(); + }) + .unwrap(); // Wait until all currently running background processes finish. db.pause_bg_work(); assert_eq!( diff --git a/tests/test_ttl.rs b/tests/test_ttl.rs index f5afe178b..16e8936c9 100644 --- a/tests/test_ttl.rs +++ b/tests/test_ttl.rs @@ -12,13 +12,12 @@ pub fn test_ttl() { let cf_opts = ColumnFamilyOptions::new(); let ttl = 10; opts.create_if_missing(true); - // cf_opts.set_num_levels(2); - // assert_eq!(2, cf_opts.get_num_levels()); + let mut db = match DB::open_cf_with_ttl( opts, path.path().to_str().unwrap(), vec![("default", cf_opts)], - &[ttl] + &[ttl], ) { Ok(db) => { println!("successfully opened db with ttl"); @@ -37,12 +36,16 @@ pub fn test_ttl() { drop(db); } - // should be able to write, read over a cf with ttl { let cf_opts = ColumnFamilyOptions::new(); let ttl = 0; - let db = match DB::open_cf_with_ttl(DBOptions::new(), path_str, vec![("cf1", cf_opts)], &[ttl]) { + let db = match DB::open_cf_with_ttl( + DBOptions::new(), + path_str, + vec![("cf1", cf_opts)], + &[ttl], + ) { Ok(db) => { println!("successfully opened cf with ttl"); db @@ -55,5 +58,4 @@ pub fn test_ttl() { let p = db.put_cf(cf1, b"k1", b"a"); assert!(p.is_ok()); } - } From 8bec72e11877b2e5fe5b0599e310f3379c35c63d Mon Sep 17 00:00:00 2001 From: chenfu Date: Wed, 17 Oct 2018 19:33:05 +0800 Subject: [PATCH 04/13] small fix to pass travis ci --- src/rocksdb.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/rocksdb.rs b/src/rocksdb.rs index 0576bd3af..f9aa51e3e 100644 --- a/src/rocksdb.rs +++ b/src/rocksdb.rs @@ -2498,8 +2498,7 @@ mod test { db1.put(b"k2", b"v2").unwrap(); db1.flush(true).unwrap(); db1.compact_range(None, None); - }) - .unwrap(); + }).unwrap(); // Wait until all currently running background processes finish. db.pause_bg_work(); assert_eq!( From 7098da9be839a17df56d8fbf9d117d00e94e2f0b Mon Sep 17 00:00:00 2001 From: chenfu Date: Thu, 18 Oct 2018 21:05:33 +0800 Subject: [PATCH 05/13] fix bug about with_ttl condition and code style --- librocksdb_sys/crocksdb/c.cc | 2 +- librocksdb_sys/crocksdb/crocksdb/c.h | 3 ++- librocksdb_sys/src/lib.rs | 2 +- src/rocksdb.rs | 33 ++++++++++++++++++---------- tests/test_ttl.rs | 6 ++--- 5 files changed, 28 insertions(+), 18 deletions(-) diff --git a/librocksdb_sys/crocksdb/c.cc b/librocksdb_sys/crocksdb/c.cc index 75471c4de..43e7f1a6e 100644 --- a/librocksdb_sys/crocksdb/c.cc +++ b/librocksdb_sys/crocksdb/c.cc @@ -689,9 +689,9 @@ crocksdb_t* crocksdb_open_column_families_with_ttl( int num_column_families, const char** column_family_names, const crocksdb_options_t** column_family_options, - crocksdb_column_family_handle_t** column_family_handles, const int32_t* ttl_array, bool read_only, + crocksdb_column_family_handle_t** column_family_handles, char** errptr) { std::vector column_families; std::vector ttls; diff --git a/librocksdb_sys/crocksdb/crocksdb/c.h b/librocksdb_sys/crocksdb/crocksdb/c.h index 208c31268..ca348e1c7 100644 --- a/librocksdb_sys/crocksdb/crocksdb/c.h +++ b/librocksdb_sys/crocksdb/crocksdb/c.h @@ -230,8 +230,9 @@ extern C_ROCKSDB_LIBRARY_API crocksdb_t* crocksdb_open_column_families_with_ttl( const crocksdb_options_t* options, const char* name, int num_column_families, const char** column_family_names, const crocksdb_options_t** column_family_options, + const int32_t* ttl_array, bool read_only, crocksdb_column_family_handle_t** column_family_handles, - const int32_t* ttl_array, bool read_only, char** errptr); + char** errptr); extern C_ROCKSDB_LIBRARY_API crocksdb_t* crocksdb_open_for_read_only_column_families( diff --git a/librocksdb_sys/src/lib.rs b/librocksdb_sys/src/lib.rs index 1b53f03df..a6da7aebb 100644 --- a/librocksdb_sys/src/lib.rs +++ b/librocksdb_sys/src/lib.rs @@ -864,9 +864,9 @@ extern "C" { num_column_families: c_int, column_family_names: *const *const c_char, column_family_options: *const *const Options, - column_family_handles: *const *mut DBCFHandle, ttl_array: *const c_int, read_only: bool, + column_family_handles: *const *mut DBCFHandle, err: *mut *mut c_char, ) -> *mut DBInstance; pub fn crocksdb_open_for_read_only_column_families( diff --git a/src/rocksdb.rs b/src/rocksdb.rs index f9aa51e3e..f3a503cb3 100644 --- a/src/rocksdb.rs +++ b/src/rocksdb.rs @@ -52,10 +52,13 @@ impl Drop for CFHandle { } } -fn ensure_default_cf_exists<'a>(list: &mut Vec>) { +fn ensure_default_cf_exists<'a>(list: &mut Vec>, ttls: &mut Vec) { let contains = list.iter().any(|ref cf| cf.is_default()); if !contains { list.push(ColumnFamilyDescriptor::default()); + if ttls.len() < list.len() { + ttls.push(0); + } } } @@ -446,7 +449,8 @@ impl DB { })?; let mut descs = cfds.into_iter().map(|t| t.into()).collect(); - ensure_default_cf_exists(&mut descs); + let mut ttls_vec = ttls.to_vec(); + ensure_default_cf_exists(&mut descs, &mut ttls_vec); let (names, options) = split_descriptors(descs); let cstrings = build_cstring_list(&names); @@ -463,11 +467,15 @@ impl DB { } else { false }; - let with_ttl = if ttls.len() > 0 && ttls.len() == cf_names.len() { - true - } else { - false - }; + let mut with_ttl = false; + + if ttls.len() > 0 { + with_ttl = true; + } + if with_ttl && ttls_vec.len() != cf_names.len() { + return Err("the length of ttls not equal to length of cfs".to_owned()); + } + let db = { let db_options = opts.inner; let db_path = cpath.as_ptr(); @@ -502,7 +510,8 @@ impl DB { } } } else { - let ttl_array = ttls.as_ptr() as *const c_int; + let ttl_array = ttls_vec.as_ptr() as *const c_int; + unsafe { ffi_try!(crocksdb_open_column_families_with_ttl( db_options, @@ -510,13 +519,14 @@ impl DB { db_cfs_count, db_cf_ptrs, db_cf_opts, - db_cf_handles, ttl_array, - readonly + readonly, + db_cf_handles )) } } }; + if cf_handles.iter().any(|h| h.is_null()) { return Err(ERR_NULL_CF_HANDLE.to_owned()); } @@ -2498,7 +2508,8 @@ mod test { db1.put(b"k2", b"v2").unwrap(); db1.flush(true).unwrap(); db1.compact_range(None, None); - }).unwrap(); + }) + .unwrap(); // Wait until all currently running background processes finish. db.pause_bg_work(); assert_eq!( diff --git a/tests/test_ttl.rs b/tests/test_ttl.rs index 16e8936c9..d91f59424 100644 --- a/tests/test_ttl.rs +++ b/tests/test_ttl.rs @@ -38,13 +38,11 @@ pub fn test_ttl() { // should be able to write, read over a cf with ttl { - let cf_opts = ColumnFamilyOptions::new(); - let ttl = 0; let db = match DB::open_cf_with_ttl( DBOptions::new(), path_str, - vec![("cf1", cf_opts)], - &[ttl], + vec![("cf1", ColumnFamilyOptions::new()), ("default", ColumnFamilyOptions::new())], + &[10,10], ) { Ok(db) => { println!("successfully opened cf with ttl"); From 749b1fb5e3aa8b07ceb94967bccf158d46afae15 Mon Sep 17 00:00:00 2001 From: chenfu Date: Thu, 18 Oct 2018 22:13:59 +0800 Subject: [PATCH 06/13] small fix to pass travis ci --- src/rocksdb.rs | 7 +++---- tests/test_ttl.rs | 7 +++++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/rocksdb.rs b/src/rocksdb.rs index f3a503cb3..dd74bf6d1 100644 --- a/src/rocksdb.rs +++ b/src/rocksdb.rs @@ -469,7 +469,7 @@ impl DB { }; let mut with_ttl = false; - if ttls.len() > 0 { + if ttls.len() > 0 { with_ttl = true; } if with_ttl && ttls_vec.len() != cf_names.len() { @@ -526,7 +526,7 @@ impl DB { } } }; - + if cf_handles.iter().any(|h| h.is_null()) { return Err(ERR_NULL_CF_HANDLE.to_owned()); } @@ -2508,8 +2508,7 @@ mod test { db1.put(b"k2", b"v2").unwrap(); db1.flush(true).unwrap(); db1.compact_range(None, None); - }) - .unwrap(); + }).unwrap(); // Wait until all currently running background processes finish. db.pause_bg_work(); assert_eq!( diff --git a/tests/test_ttl.rs b/tests/test_ttl.rs index d91f59424..f69c2ebf3 100644 --- a/tests/test_ttl.rs +++ b/tests/test_ttl.rs @@ -41,8 +41,11 @@ pub fn test_ttl() { let db = match DB::open_cf_with_ttl( DBOptions::new(), path_str, - vec![("cf1", ColumnFamilyOptions::new()), ("default", ColumnFamilyOptions::new())], - &[10,10], + vec![ + ("cf1", ColumnFamilyOptions::new()), + ("default", ColumnFamilyOptions::new()) + ], + &[10, 10], ) { Ok(db) => { println!("successfully opened cf with ttl"); From 20f0860e8ef5de1aee0b438f2b62dbb888bb907a Mon Sep 17 00:00:00 2001 From: chenfu Date: Thu, 18 Oct 2018 22:19:23 +0800 Subject: [PATCH 07/13] small fix to pass travis ci --- tests/test_ttl.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_ttl.rs b/tests/test_ttl.rs index f69c2ebf3..6fb317309 100644 --- a/tests/test_ttl.rs +++ b/tests/test_ttl.rs @@ -43,7 +43,7 @@ pub fn test_ttl() { path_str, vec![ ("cf1", ColumnFamilyOptions::new()), - ("default", ColumnFamilyOptions::new()) + ("default", ColumnFamilyOptions::new()), ], &[10, 10], ) { From c3918a55410a94a9d6a767bf7bd0aab57d24e703 Mon Sep 17 00:00:00 2001 From: chenfu Date: Tue, 23 Oct 2018 14:40:13 +0800 Subject: [PATCH 08/13] unify the with_ttl funtion's conditon --- src/rocksdb.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/rocksdb.rs b/src/rocksdb.rs index dd74bf6d1..29bc319f0 100644 --- a/src/rocksdb.rs +++ b/src/rocksdb.rs @@ -375,11 +375,10 @@ impl DB { pub fn open_with_ttl(opts: DBOptions, path: &str, ttls: &[i32]) -> Result { let cfds: Vec<&str> = vec![]; - if ttls.len() > 0 { - DB::open_cf_with_ttl(opts, path, cfds, ttls) - } else { - DB::open_cf(opts, path, cfds) + if ttls.len() == 0 { + return Err("ttls is empty in with_ttl function".to_owned()); } + DB::open_cf_with_ttl(opts, path, cfds, ttls) } pub fn open_cf<'a, T>(opts: DBOptions, path: &str, cfds: Vec) -> Result @@ -398,6 +397,9 @@ impl DB { where T: Into>, { + if ttls.len() == 0 { + return Err("ttls is empty in with_ttl function".to_owned()); + } DB::open_cf_internal(opts, path, cfds, ttls, None) } From d827ba755cdbf2560cdecc15f1684d1cab43aafa Mon Sep 17 00:00:00 2001 From: chenfu Date: Tue, 23 Oct 2018 16:57:35 +0800 Subject: [PATCH 09/13] fix condition about with_ttl and add more tests --- src/rocksdb.rs | 17 ++++++---- tests/test_ttl.rs | 81 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 90 insertions(+), 8 deletions(-) diff --git a/src/rocksdb.rs b/src/rocksdb.rs index 29bc319f0..9bfa7b825 100644 --- a/src/rocksdb.rs +++ b/src/rocksdb.rs @@ -450,6 +450,7 @@ impl DB { ) })?; + let cfds_len = cfds.len(); let mut descs = cfds.into_iter().map(|t| t.into()).collect(); let mut ttls_vec = ttls.to_vec(); ensure_default_cf_exists(&mut descs, &mut ttls_vec); @@ -469,14 +470,16 @@ impl DB { } else { false }; - let mut with_ttl = false; - if ttls.len() > 0 { - with_ttl = true; - } - if with_ttl && ttls_vec.len() != cf_names.len() { - return Err("the length of ttls not equal to length of cfs".to_owned()); - } + let with_ttl = if ttls.len() > 0 { + if ttls.len() == cfds_len && ttls_vec.len() == cf_names.len() { + true + } else { + return Err("the length of ttls not equal to length of cfs".to_owned()); + } + } else { + false + }; let db = { let db_options = opts.inner; diff --git a/tests/test_ttl.rs b/tests/test_ttl.rs index 6fb317309..194295369 100644 --- a/tests/test_ttl.rs +++ b/tests/test_ttl.rs @@ -33,18 +33,52 @@ pub fn test_ttl() { } } assert_eq!(db.cf_names(), vec!["cf1", "default"]); + + match db.create_cf("cf2") { + Ok(_) => println!("cf2 created successfully"), + Err(e) => { + panic!("could not create column family: {}", e); + } + } + assert_eq!(db.cf_names(), vec!["cf1","cf2", "default"]); drop(db); } - // should be able to write, read over a cf with ttl + // should be able to write, read over a cf with the length of ttls equals to that of cfs { let db = match DB::open_cf_with_ttl( DBOptions::new(), path_str, vec![ ("cf1", ColumnFamilyOptions::new()), + ("cf2", ColumnFamilyOptions::new()), ("default", ColumnFamilyOptions::new()), ], + &[10, 10, 10], + ) { + Ok(db) => { + println!("successfully opened cf with ttl"); + db + } + Err(e) => panic!("failed to open cf with ttl: {}", e), + }; + let cf1 = db.cf_handle("cf1").unwrap(); + assert!(db.put_cf(cf1, b"k1", b"v1").is_ok()); + assert!(db.get_cf(cf1, b"k1").unwrap().unwrap().to_utf8().unwrap() == "v1"); + let p = db.put_cf(cf1, b"k1", b"a"); + assert!(p.is_ok()); + } + + // should be able to write, read over a cf with the length of ttls equals to that of cfs. + // default cf could be with ttl 0 if it is not in cfds + { + let db = match DB::open_cf_with_ttl( + DBOptions::new(), + path_str, + vec![ + ("cf1", ColumnFamilyOptions::new()), + ("cf2", ColumnFamilyOptions::new()), + ], &[10, 10], ) { Ok(db) => { @@ -59,4 +93,49 @@ pub fn test_ttl() { let p = db.put_cf(cf1, b"k1", b"a"); assert!(p.is_ok()); } + + // should fail to open cf with ttl when the length of ttls not equal to that of cfs + { + let _db = match DB::open_cf_with_ttl( + DBOptions::new(), + path_str, + vec![ + ("cf1", ColumnFamilyOptions::new()), + ("cf2", ColumnFamilyOptions::new()), + ], + &[10], + ) { + Ok(_) => panic!( + "should not have opened DB successfully with ttl \ + when the length of ttl not equal to that of cfs" + ), + Err(e) => assert!(e.starts_with( + "the length of ttls not equal to length of cfs" + )), + }; + } + + // should fail to open cf with ttl when the length of ttls not equal to that of cfs + // when default is in cfds, it's ttl must be supplied + { + let _db = match DB::open_cf_with_ttl( + DBOptions::new(), + path_str, + vec![ + ("cf1", ColumnFamilyOptions::new()), + ("cf2", ColumnFamilyOptions::new()), + ("default", ColumnFamilyOptions::new()), + ], + &[10,10], + ) { + Ok(_) => panic!( + "should not have opened DB successfully with ttl \ + when the length of ttl not equal to that of cfs" + ), + Err(e) => assert!(e.starts_with( + "the length of ttls not equal to length of cfs" + )), + }; + } + } From beb6990e141c04712ba657a1e191b81b75b50d24 Mon Sep 17 00:00:00 2001 From: chenfu Date: Tue, 23 Oct 2018 17:34:32 +0800 Subject: [PATCH 10/13] little fix to pass travis ci --- tests/test_ttl.rs | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/tests/test_ttl.rs b/tests/test_ttl.rs index 194295369..56dcb7db2 100644 --- a/tests/test_ttl.rs +++ b/tests/test_ttl.rs @@ -40,7 +40,7 @@ pub fn test_ttl() { panic!("could not create column family: {}", e); } } - assert_eq!(db.cf_names(), vec!["cf1","cf2", "default"]); + assert_eq!(db.cf_names(), vec!["cf1", "cf2", "default"]); drop(db); } @@ -107,11 +107,9 @@ pub fn test_ttl() { ) { Ok(_) => panic!( "should not have opened DB successfully with ttl \ - when the length of ttl not equal to that of cfs" + when the length of ttl not equal to that of cfs" ), - Err(e) => assert!(e.starts_with( - "the length of ttls not equal to length of cfs" - )), + Err(e) => assert!(e.starts_with("the length of ttls not equal to length of cfs")), }; } @@ -126,15 +124,13 @@ pub fn test_ttl() { ("cf2", ColumnFamilyOptions::new()), ("default", ColumnFamilyOptions::new()), ], - &[10,10], + &[10, 10], ) { Ok(_) => panic!( "should not have opened DB successfully with ttl \ - when the length of ttl not equal to that of cfs" + when the length of ttl not equal to that of cfs" ), - Err(e) => assert!(e.starts_with( - "the length of ttls not equal to length of cfs" - )), + Err(e) => assert!(e.starts_with("the length of ttls not equal to length of cfs")), }; } From 451f64d8fa8a5df0b47fd52e87418e3d08b6f47a Mon Sep 17 00:00:00 2001 From: chenfu Date: Tue, 23 Oct 2018 18:04:41 +0800 Subject: [PATCH 11/13] little fix to pass travis ci --- tests/test_ttl.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_ttl.rs b/tests/test_ttl.rs index 56dcb7db2..1590ef691 100644 --- a/tests/test_ttl.rs +++ b/tests/test_ttl.rs @@ -133,5 +133,4 @@ pub fn test_ttl() { Err(e) => assert!(e.starts_with("the length of ttls not equal to length of cfs")), }; } - } From be60a5fb6ece426456303eb3373786e8ba21a13b Mon Sep 17 00:00:00 2001 From: chenfu Date: Wed, 24 Oct 2018 14:11:45 +0800 Subject: [PATCH 12/13] fix with_ttl condition and make it more clear --- src/rocksdb.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/rocksdb.rs b/src/rocksdb.rs index 9bfa7b825..27b3a90e7 100644 --- a/src/rocksdb.rs +++ b/src/rocksdb.rs @@ -56,7 +56,7 @@ fn ensure_default_cf_exists<'a>(list: &mut Vec>, ttls let contains = list.iter().any(|ref cf| cf.is_default()); if !contains { list.push(ColumnFamilyDescriptor::default()); - if ttls.len() < list.len() { + if ttls.len() > 0 { ttls.push(0); } } @@ -450,7 +450,7 @@ impl DB { ) })?; - let cfds_len = cfds.len(); + let mut descs = cfds.into_iter().map(|t| t.into()).collect(); let mut ttls_vec = ttls.to_vec(); ensure_default_cf_exists(&mut descs, &mut ttls_vec); @@ -471,8 +471,8 @@ impl DB { false }; - let with_ttl = if ttls.len() > 0 { - if ttls.len() == cfds_len && ttls_vec.len() == cf_names.len() { + let with_ttl = if ttls_vec.len() > 0 { + if ttls_vec.len() == cf_names.len() { true } else { return Err("the length of ttls not equal to length of cfs".to_owned()); From 987684e6e10d6f328ba76e612bdf0abe867217dc Mon Sep 17 00:00:00 2001 From: chenfu Date: Wed, 24 Oct 2018 14:17:41 +0800 Subject: [PATCH 13/13] litte fix to pass travis ci --- src/rocksdb.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/rocksdb.rs b/src/rocksdb.rs index 27b3a90e7..b3f97d922 100644 --- a/src/rocksdb.rs +++ b/src/rocksdb.rs @@ -450,7 +450,6 @@ impl DB { ) })?; - let mut descs = cfds.into_iter().map(|t| t.into()).collect(); let mut ttls_vec = ttls.to_vec(); ensure_default_cf_exists(&mut descs, &mut ttls_vec);