diff --git a/.gitignore b/.gitignore index f366f88b3..7d871c038 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,4 @@ compile_commands.json *.cmd core +CMAKE_BINARY_DIR diff --git a/.travis.yml b/.travis.yml index ee4a96f65..76d2a773d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,7 +46,18 @@ matrix: - gcc-4.9 - libsubunit-dev - # gcc 5 on linux + # gcc 5 on linux *without rust* + # + - env: + - C_COMPILER=gcc-5 + addons: + apt: + <<: *apt + packages: + - gcc-5 + - libsubunit-dev + + # gcc 5 on linux *with* rust - env: - C_COMPILER=gcc-5 - RUST_ENABLED=1 diff --git a/ci/before-install.sh b/ci/before-install.sh index d948672db..5916df4c6 100755 --- a/ci/before-install.sh +++ b/ci/before-install.sh @@ -9,7 +9,9 @@ TEMP="$(mktemp -d -t TEMP.XXXXXXX)" || die "failed to make tmpdir" cleanup() { [[ -n "${TEMP:-}" ]] && rm -rf "${TEMP}"; } trap cleanup EXIT -TOPLEVEL="$(git -C "$(cd "$(dirname "$0")" >/dev/null || exit 1; pwd)" rev-parse --show-toplevel)" || die 'failed to find TOPLEVEL' +realpath() { python -c "import os,sys; print os.path.realpath(sys.argv[1])" "$1"; } + +TOPLEVEL="$(cd "$(dirname "$(realpath "$0" >/dev/null || exit 1)")" && git rev-parse --show-toplevel)" || die 'failed to find TOPLEVEL' # for osx: 0. update brew; 1. install cmake if missing; 2. (gcc) unlink pre-installed gcc; 3. (gcc) install desired version of gcc diff --git a/ci/install-check.sh b/ci/install-check.sh index 628d6d229..59f108200 100755 --- a/ci/install-check.sh +++ b/ci/install-check.sh @@ -17,7 +17,9 @@ TEMP="$(mktemp -d -t TEMP.XXXXXXX)" || die "failed to make tmpdir" cleanup() { [[ -n "${TEMP:-}" ]] && rm -rf "${TEMP}"; } trap cleanup EXIT -TOPLEVEL="$(git -C "$(cd "$(dirname "$0")" >/dev/null || exit 1; pwd)" rev-parse --show-toplevel)" || die 'failed to find TOPLEVEL' +realpath() { python -c "import os,sys; print os.path.realpath(sys.argv[1])" "$1"; } + +TOPLEVEL="$(cd "$(dirname "$(realpath "$0" >/dev/null || exit 1)")" && git rev-parse --show-toplevel)" || die 'failed to find TOPLEVEL' CHECK_VERSION=0.12.0 CHECK_TARBALL="check-${CHECK_VERSION}.tar.gz" diff --git a/ci/local-clean-build-and-test.sh b/ci/local-clean-build-and-test.sh new file mode 100755 index 000000000..b5f0bbcf6 --- /dev/null +++ b/ci/local-clean-build-and-test.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -euo pipefail +IFS=$'\n\t' + +die() { echo "fatal: $*" >&2; exit 1; } + +TEMP="$(mktemp -d -t TEMP.XXXXXXX)" || die "failed to make tmpdir" +cleanup() { [[ -n "${TEMP:-}" ]] && rm -rf "${TEMP}"; } +trap cleanup EXIT + + +if [[ -n "$(git status --porcelain)" ]]; then + die "dirty working copy state, please commit changes before running this script" +fi + +git archive --format=tar --prefix=pelikan/ HEAD .|tar -C "$TEMP" -xf- +cd "$TEMP/pelikan" + +bash ./ci/run.sh diff --git a/ci/run.sh b/ci/run.sh index 77cbcdd1e..a147d6fa6 100755 --- a/ci/run.sh +++ b/ci/run.sh @@ -15,11 +15,12 @@ export PATH=$HOME/.cargo/bin:$PATH CMAKE_ARGS=( -DBUILD_AND_INSTALL_CHECK=yes - -DTARGET_CDB=yes - -DHAVE_RUST=yes - -DRUST_VERBOSE_BUILD=1 ) +if [[ -n "${RUST_ENABLED:-}" ]]; then + CMAKE_ARGS+=( -DTARGET_CDB=yes -DHAVE_RUST=yes -DRUST_VERBOSE_BUILD=1 ) +fi + # TODO: run cmake3 on centos hosts mkdir -p _build && ( cd _build && cmake ${CMAKE_ARGS[@]} .. && make -j && make check ) || die 'make failed' @@ -30,18 +31,12 @@ egrep -r ":F:|:E:" . |grep -v 'Binary file' || true set +e -( cd src/storage/cdb && env RUST_BACKTRACE=full cargo test ) +if [[ -n "${RUST_ENABLED:-}" ]]; then + ( cd src/storage/cdb && env RUST_BACKTRACE=full cargo test ) +fi RESULT=$? -if [[ "$(uname -s)" == "Darwin" ]]; then - if [[ $RESULT -ne 0 ]]; then - echo "Rust test failed on OSX, but this does not fail the build" >&2 - fi - - exit 0 -fi - if [[ $RESULT -ne 0 ]]; then echo "Build failure" >&2 exit $RESULT diff --git a/cmake/CMakeCargo.cmake b/cmake/CMakeCargo.cmake index 975b0ccd2..8d7fc1b2f 100644 --- a/cmake/CMakeCargo.cmake +++ b/cmake/CMakeCargo.cmake @@ -38,7 +38,8 @@ function(cargo_build) cmake_parse_arguments(CARGO "" "NAME" "" ${ARGN}) string(REPLACE "-" "_" LIB_NAME ${CARGO_NAME}) - get_filename_component(CARGO_TARGET_DIR "${CMAKE_CURRENT_BINARY_DIR}" ABSOLUTE) + get_filename_component(CARGO_TARGET_DIR "${CMAKE_BINARY_DIR}/target" ABSOLUTE) + message(STATUS "CARGO_TARGET_DIR ${CARGO_TARGET_DIR}") cargo_set_lib_target(LIB_NAME) diff --git a/deps/ccommon/ci/before-install.sh b/deps/ccommon/ci/before-install.sh index cff9327c3..8705a5005 100755 --- a/deps/ccommon/ci/before-install.sh +++ b/deps/ccommon/ci/before-install.sh @@ -9,7 +9,10 @@ TEMP="$(mktemp -d -t TEMP.XXXXXXX)" || die "failed to make tmpdir" cleanup() { [[ -n "${TEMP:-}" ]] && rm -rf "${TEMP}"; } trap cleanup EXIT -TOPLEVEL="$(git -C "$(cd "$(dirname "$0")" >/dev/null || exit 1; pwd)" rev-parse --show-toplevel)" || die 'failed to find TOPLEVEL' +realpath() { python -c "import os,sys; print os.path.realpath(sys.argv[1])" "$1"; } + +TOPLEVEL="$(cd "$(dirname "$(realpath "$0" >/dev/null || exit 1)")" && git rev-parse --show-toplevel)" || die 'failed to find TOPLEVEL' + # for osx: 0. update brew; 1. install cmake if missing; 2. (gcc) unlink pre-installed gcc; 3. (gcc) install desired version of gcc diff --git a/deps/ccommon/ci/install-check.sh b/deps/ccommon/ci/install-check.sh index 628d6d229..a69c317e3 100755 --- a/deps/ccommon/ci/install-check.sh +++ b/deps/ccommon/ci/install-check.sh @@ -17,7 +17,10 @@ TEMP="$(mktemp -d -t TEMP.XXXXXXX)" || die "failed to make tmpdir" cleanup() { [[ -n "${TEMP:-}" ]] && rm -rf "${TEMP}"; } trap cleanup EXIT -TOPLEVEL="$(git -C "$(cd "$(dirname "$0")" >/dev/null || exit 1; pwd)" rev-parse --show-toplevel)" || die 'failed to find TOPLEVEL' +realpath() { python -c "import os,sys; print os.path.realpath(sys.argv[1])" "$1"; } + +TOPLEVEL="$(cd "$(dirname "$(realpath "$0" >/dev/null || exit 1)")" && git rev-parse --show-toplevel)" || die 'failed to find TOPLEVEL' + CHECK_VERSION=0.12.0 CHECK_TARBALL="check-${CHECK_VERSION}.tar.gz" diff --git a/deps/ccommon/rust/ccommon_rs/src/bstring.rs b/deps/ccommon/rust/ccommon_rs/src/bstring.rs index 88c56f510..9e057bb07 100644 --- a/deps/ccommon/rust/ccommon_rs/src/bstring.rs +++ b/deps/ccommon/rust/ccommon_rs/src/bstring.rs @@ -95,8 +95,8 @@ impl BStr { // it's ok to ignore the cast_ptr_alignment lint because we're going // *mut CCbstring -> &BStr -> *mut CCbstring - #[allow(cast_ptr_alignment)] #[allow(unknown_lints)] + #[allow(cast_ptr_alignment)] #[inline] pub fn as_ptr(&self) -> *mut CCbstring { (&*self) as *const _ as *mut _ diff --git a/src/server/cdb/CMakeLists.txt b/src/server/cdb/CMakeLists.txt index 6e54e7ca0..58542aeab 100644 --- a/src/server/cdb/CMakeLists.txt +++ b/src/server/cdb/CMakeLists.txt @@ -8,7 +8,7 @@ set(SOURCE stats.c) set(MODULES - cdb_ffi_static + cdb_rs_static core protocol_admin protocol_memcache @@ -25,7 +25,7 @@ set(TARGET_NAME ${PROJECT_NAME}_cdb) add_executable(${TARGET_NAME} ${SOURCE}) target_link_libraries(${TARGET_NAME} ${MODULES} ${LIBS}) -add_dependencies(${TARGET_NAME} cdb_ffi_static) +add_dependencies(${TARGET_NAME} cdb_rs_static) install(TARGETS ${TARGET_NAME} RUNTIME DESTINATION bin) add_dependencies(service ${TARGET_NAME}) diff --git a/src/server/cdb/data/process.c b/src/server/cdb/data/process.c index 52b0b1f9c..629caef69 100644 --- a/src/server/cdb/data/process.c +++ b/src/server/cdb/data/process.c @@ -3,7 +3,7 @@ #include "process.h" #include "protocol/data/memcache_include.h" -#include "storage/cdb/cdb.h" +#include "storage/cdb/cdb_rs.h" #include #include @@ -26,10 +26,10 @@ static struct bstring value_buf; static bool process_init = false; static process_metrics_st *process_metrics = NULL; -static struct CDBHandle *cdb_handle = NULL; +static struct cdb_handle *cdb_handle = NULL; void -process_setup(process_options_st *options, process_metrics_st *metrics, struct CDBHandle *handle) +process_setup(process_options_st *options, process_metrics_st *metrics, struct cdb_handle *handle) { log_info("set up the %s module", CDB_PROCESS_MODULE_NAME); @@ -68,15 +68,12 @@ process_teardown(void) } if (cdb_handle != NULL) { - struct CDBHandle *p = cdb_handle; - cdb_handle = NULL; - cdb_handle_destroy(p); + cdb_handle_destroy(&cdb_handle); } if (value_buf.data != NULL) { char *p = value_buf.data; - value_buf.data = NULL; - value_buf.len = 0; + bstring_init(&value_buf); cc_free(p); } @@ -101,8 +98,7 @@ _get_key(struct response *rsp, struct bstring *key) rsp->key = *key; rsp->flag = 0; rsp->vcas = 0; - rsp->vstr.len = vstr->len; - rsp->vstr.data = vstr->data; + rsp->vstr = *vstr; log_verb("found key at %p, location %p", key, vstr); return true; diff --git a/src/server/cdb/data/process.h b/src/server/cdb/data/process.h index 8b7cd8df7..8e12b1c93 100644 --- a/src/server/cdb/data/process.h +++ b/src/server/cdb/data/process.h @@ -1,6 +1,6 @@ #pragma once -#include "storage/cdb/cdb.h" +#include "storage/cdb/cdb_rs.h" #include #include @@ -33,7 +33,7 @@ typedef struct { PROCESS_METRIC(METRIC_DECLARE) } process_metrics_st; -void process_setup(process_options_st *options, process_metrics_st *metrics, struct CDBHandle *cdb_handle); +void process_setup(process_options_st *options, process_metrics_st *metrics, struct cdb_handle *cdb_handle); void process_teardown(void); int cdb_process_read(struct buf **rbuf, struct buf **wbuf, void **data); diff --git a/src/server/cdb/main.c b/src/server/cdb/main.c index 56389669c..60ba740a2 100644 --- a/src/server/cdb/main.c +++ b/src/server/cdb/main.c @@ -1,7 +1,7 @@ #include "admin/process.h" #include "setting.h" #include "stats.h" -#include "storage/cdb/cdb.h" +#include "storage/cdb/cdb_rs.h" #include "time/time.h" #include "util/util.h" @@ -80,17 +80,24 @@ teardown(void) log_teardown(); } -static struct CDBHandle* +static struct cdb_handle * setup_cdb_handle(cdb_options_st *opt) { cdb_setup(); - char *cdb_file_path = opt->cdb_file_path.val.vstr; - if (cdb_file_path == NULL) { + struct cdb_handle_create_config cfg; + bstring_init(cfg.path); + + cfg.load_method = (opt->use_mmap.val.vbool) ? CDB_MMAP : CDB_HEAP; + + bstring_set_text(cfg.path, opt->cdb_file_path.val.vstr); + + if (cfg.path == NULL) { log_stderr("cdb_file_path option not set, cannot continue"); exit(EX_CONFIG); } - return cdb_handle_create(cdb_file_path); + + return cdb_handle_create(&cfg); } static void @@ -140,7 +147,7 @@ setup(void) compose_setup(NULL, &stats.compose_rsp); klog_setup(&setting.klog, &stats.klog); - struct CDBHandle* cdb_handle = setup_cdb_handle(&setting.cdb); + struct cdb_handle *cdb_handle = setup_cdb_handle(&setting.cdb); if (cdb_handle == NULL) { log_stderr("failed to set up cdb"); goto error; diff --git a/src/server/cdb/setting.h b/src/server/cdb/setting.h index 431f4e13d..b4ac85c3e 100644 --- a/src/server/cdb/setting.h +++ b/src/server/cdb/setting.h @@ -16,14 +16,15 @@ #include #include -/* option related */ -/* name type default description */ -#define CDB_OPTION(ACTION) \ - ACTION( daemonize, OPTION_TYPE_BOOL, false, "daemonize the process" )\ - ACTION( pid_filename, OPTION_TYPE_STR, NULL, "file storing the pid" )\ - ACTION( cdb_file_path, OPTION_TYPE_STR, "db.cdb", "location of the .cdb file" )\ - ACTION( dlog_intvl, OPTION_TYPE_UINT, 500, "debug log flush interval(ms)" )\ - ACTION( klog_intvl, OPTION_TYPE_UINT, 100, "cmd log flush interval(ms)" ) +/* option related */ +/* name type default description */ +#define CDB_OPTION(ACTION) \ + ACTION( daemonize, OPTION_TYPE_BOOL, false, "daemonize the process" )\ + ACTION( pid_filename, OPTION_TYPE_STR, NULL, "file storing the pid" )\ + ACTION( cdb_file_path, OPTION_TYPE_STR, "db.cdb", "location of the .cdb file" )\ + ACTION( use_mmap, OPTION_TYPE_BOOL, false, "use mmap to load the file, false: use the heap" )\ + ACTION( dlog_intvl, OPTION_TYPE_UINT, 500, "debug log flush interval(ms)" )\ + ACTION( klog_intvl, OPTION_TYPE_UINT, 100, "cmd log flush interval(ms)" ) typedef struct { CDB_OPTION(OPTION_DECLARE) diff --git a/src/storage/cdb/.gitignore b/src/storage/cdb/.gitignore index ab4c6da5b..35d3e3e34 100644 --- a/src/storage/cdb/.gitignore +++ b/src/storage/cdb/.gitignore @@ -5,3 +5,5 @@ *.cdb perf.data* *.cdb* +.cargo/ +dist/ diff --git a/src/storage/cdb/CMakeLists.txt b/src/storage/cdb/CMakeLists.txt index 17fd15ef1..18e4b5f9e 100644 --- a/src/storage/cdb/CMakeLists.txt +++ b/src/storage/cdb/CMakeLists.txt @@ -1,2 +1,5 @@ -add_subdirectory(cdb_ffi) +execute_process(COMMAND + env "CMAKE_BINARY_DIR=${CMAKE_BINARY_DIR}" bash scripts/build-cargo-config.sh + WORKING_DIRECTORY "$CMAKE_CURRENT_LIST_DIR}") +add_subdirectory(cdb_rs) add_dependencies(ccommon_rs_static_target ccommon-static) diff --git a/src/storage/cdb/Cargo.lock b/src/storage/cdb/Cargo.lock index 9d3c64a9c..5d3952238 100644 --- a/src/storage/cdb/Cargo.lock +++ b/src/storage/cdb/Cargo.lock @@ -1,6 +1,6 @@ [[package]] name = "aho-corasick" -version = "0.6.4" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -16,7 +16,7 @@ dependencies = [ [[package]] name = "atty" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", @@ -30,7 +30,7 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "backtrace-sys 0.1.23 (registry+https://github.com/rust-lang/crates.io-index)", - "cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-demangle 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -41,26 +41,26 @@ name = "backtrace-sys" version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cc 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "bindgen" -version = "0.37.0" +version = "0.37.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cexpr 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "clang-sys 0.22.0 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "clang-sys 0.23.0 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)", - "env_logger 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "peeking_take_while 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "which 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -89,7 +89,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "bytes" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -98,14 +98,14 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "cc_binding" version = "0.1.0" dependencies = [ - "bindgen 0.37.0 (registry+https://github.com/rust-lang/crates.io-index)", + "bindgen 0.37.4 (registry+https://github.com/rust-lang/crates.io-index)", "failure 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -116,36 +116,36 @@ dependencies = [ "cc_binding 0.1.0", "failure 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "cdb_ffi" -version = "0.2.0" +name = "cdb_rs" +version = "0.1.0" dependencies = [ - "bindgen 0.37.0 (registry+https://github.com/rust-lang/crates.io-index)", - "bytes 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "bindgen 0.37.4 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)", "cc_binding 0.1.0", "ccommon_rs 0.1.0", - "cdb_rs 0.1.0", - "env_logger 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)", + "clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)", + "failure 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "failure_derive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", + "proptest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", + "tempfile 3.0.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "cdb_rs" +name = "cdbgen" version = "0.1.0" dependencies = [ - "bytes 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "cdb_rs 0.1.0", "clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)", - "env_logger 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", - "proptest 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tempfile 3.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tinycdb 0.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "tempfile 3.0.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -158,12 +158,12 @@ dependencies = [ [[package]] name = "cfg-if" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "clang-sys" -version = "0.22.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", @@ -177,7 +177,7 @@ version = "2.31.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", - "atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", + "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -195,14 +195,14 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.5.10" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", + "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "termcolor 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -267,7 +267,7 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -280,16 +280,16 @@ name = "libloading" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cc 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "log" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -327,7 +327,7 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -345,18 +345,18 @@ dependencies = [ [[package]] name = "proptest" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bit-set 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "regex-syntax 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rusty-fork 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "tempfile 3.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tempfile 3.0.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -389,7 +389,7 @@ dependencies = [ [[package]] name = "rand" -version = "0.5.2" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -419,24 +419,12 @@ dependencies = [ [[package]] name = "regex" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", - "thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "regex" -version = "1.0.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", + "aho-corasick 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", "thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -448,15 +436,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "regex-syntax" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "regex-syntax" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -482,7 +462,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tempfile 3.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tempfile 3.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "wait-timeout 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -520,11 +500,11 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.0.2" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", "remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -532,10 +512,10 @@ dependencies = [ [[package]] name = "termcolor" -version = "0.3.6" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "wincolor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "wincolor 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -561,27 +541,10 @@ name = "thread_local" version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "tinycdb" -version = "0.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", - "tinycdb-sys 0.0.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "tinycdb-sys" -version = "0.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "ucd-util" version = "0.1.1" @@ -667,31 +630,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "wincolor" -version = "0.1.6" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", ] [metadata] -"checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4" +"checksum aho-corasick 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c1c6d463cbe7ed28720b5b489e7c083eeb8f90d08be2a0d6bb9e1ffea9ce1afa" "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" -"checksum atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "2fc4a1aa4c24c0718a250f0681885c1af91419d242f29eb8f2ab28502d80dbd1" +"checksum atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652" "checksum backtrace 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "89a47830402e9981c5c41223151efcced65a0510c13097c769cede7efb34782a" "checksum backtrace-sys 0.1.23 (registry+https://github.com/rust-lang/crates.io-index)" = "bff67d0c06556c0b8e6b5f090f0eac52d950d9dfd1d35ba04e4ca3543eaf6a7e" -"checksum bindgen 0.37.0 (registry+https://github.com/rust-lang/crates.io-index)" = "429d032f8d65efdf12b2b799c84e339de7fe8e173e539911863887a935d202e9" +"checksum bindgen 0.37.4 (registry+https://github.com/rust-lang/crates.io-index)" = "1b25ab82877ea8fe6ce1ce1f8ac54361f0218bad900af9eb11803994bf67c221" "checksum bit-set 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6f1efcc46c18245a69c38fcc5cc650f16d3a59d034f3106e9ed63748f695730a" "checksum bit-vec 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4440d5cb623bb7390ae27fec0bb6c61111969860f8e3ae198bfa0663645e67cf" "checksum bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d0c54bb8f454c567f21197eefcdbf5679d0bd99f2ddbe52e84c77061952e6789" "checksum byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "74c0b906e9446b0a2e4f760cdb3fa4b2c48cdc6db8766a845c54b6ff063fd2e9" -"checksum bytes 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7dd32989a66957d3f0cba6588f15d4281a733f4e9ffc43fcd2385f57d3bf99ff" -"checksum cc 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)" = "49ec142f5768efb5b7622aebc3fdbdbb8950a4b9ba996393cb76ef7466e8747d" +"checksum bytes 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e178b8e0e239e844b083d5a0d4a156b2654e67f9f80144d48398fcd736a24fb8" +"checksum cc 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)" = "2119ea4867bd2b8ed3aecab467709720b2d55b1bcfe09f772fd68066eaf15275" "checksum cexpr 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "42aac45e9567d97474a834efdee3081b3c942b2205be932092f53354ce503d6c" -"checksum cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "405216fd8fe65f718daa7102ea808a946b6ce40c742998fbfd3463645552de18" -"checksum clang-sys 0.22.0 (registry+https://github.com/rust-lang/crates.io-index)" = "939a1a34310b120d26eba35c29475933128b0ec58e24b43327f8dbe6036fc538" +"checksum cfg-if 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "efe5c877e17a9c717a0bf3613b2709f723202c4e4675cc8f12926ded29bcb17e" +"checksum clang-sys 0.23.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d7f7c04e52c35222fffcc3a115b5daf5f7e2bfb71c13c4e2321afe1fc71859c2" "checksum clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f0f16b89cbb9ee36d87483dc939fe9f1e13c05898d56d7b230a0d4dff033a536" "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" -"checksum env_logger 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)" = "0e6e40ebb0e66918a37b38c7acab4e10d299e0463fe2af5d29b9cc86710cfd2a" +"checksum env_logger 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)" = "7873e292d20e8778f951278972596b3df36ac72a65c5b406f6d4961070a870c1" "checksum failure 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "934799b6c1de475a012a02dab0ace1ace43789ee4b99bcfbf1a2e3e8ced5de82" "checksum failure_derive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c7cdda555bb90c9bb67a3b670a0f42de8e73f5981524123ad8578aafec8ddb8b" "checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" @@ -700,31 +663,29 @@ dependencies = [ "checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb" "checksum humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0484fda3e7007f2a4a0d9c3a703ca38c71c54c55602ce4660c419fd32e188c9e" "checksum iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dbe6e417e7d0975db6512b90796e8ce223145ac4e33c377e4a42882a0e88bb08" -"checksum lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e6412c5e2ad9584b0b8e979393122026cdd6d2a80b933f890dcd694ddbe73739" +"checksum lazy_static 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "fb497c35d362b6a331cfd94956a07fc2c78a4604cdbee844a81170386b996dd3" "checksum libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" = "b685088df2b950fccadf07a7187c8ef846a959c142338a48f9dc0b94517eb5f1" "checksum libloading 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9c3ad660d7cb8c5822cd83d10897b0f1f1526792737a179e73896152f85b88c2" -"checksum log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6fddaa003a65722a7fb9e26b0ce95921fe4ba590542ced664d8ce2fa26f9f3ac" +"checksum log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "61bd98ae7f7b754bc53dca7d44b604f733c6bba044ea6f41bc8d89272d8161d2" "checksum memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "148fab2e51b4f1cfc66da2a7c32981d1d3c083a803978268bb11fe4b86925e7a" "checksum memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "796fba70e76612589ed2ce7f45282f5af869e0fdd7cc6199fa1aa1f1d591ba9d" "checksum memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e2ffa2c986de11a9df78620c01eeaaf27d94d3ff02bf81bfcca953102dd0c6ff" "checksum nom 3.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05aec50c70fd288702bcd93284a8444607f3292dbdf2a30de5ea5dcdbe72287b" -"checksum num-traits 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "775393e285254d2f5004596d69bb8bc1149754570dcc08cf30cabeba67955e28" +"checksum num-traits 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "630de1ef5cc79d0cdd78b7e33b81f083cbfe90de0f4b2b2f07f905867c70e9fe" "checksum peeking_take_while 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" "checksum proc-macro2 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "77997c53ae6edd6d187fec07ec41b207063b5ee6f33680e9fa86d405cdd313d4" -"checksum proptest 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2b21129762855f6e5d39628cab74d9040d63bc91edc343112d22582c9bb87a9c" +"checksum proptest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "27f275a76b824714046ce0b1e00323e06437e027f2d31b2b6272cae30afaf18d" "checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0" "checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" "checksum quote 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9949cfe66888ffe1d53e6ec9d9f3b70714083854be20fd5e271b232a017401e8" "checksum rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "eba5f8cb59cc50ed56be8880a5c7b496bfd9bd26394e176bc67884094145c2c5" -"checksum rand 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d0d9f869af32e387d9e0f2bdb64326b8ac84c81d5e55459d0bc7526b0fdb3671" +"checksum rand 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)" = "12397506224b2f93e6664ffc4f664b29be8208e5157d3d90b44f09b5fae470ea" "checksum rand_core 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "edecf0f94da5551fc9b492093e30b041a891657db7940ee221f9d2f66e82eef2" "checksum redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "c214e91d3ecf43e9a4e41e578973adeb14b474f2bee858742d127af75a0112b1" "checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" -"checksum regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9329abc99e39129fcceabd24cf5d85b4671ef7c29c50e972bc5afe32438ec384" -"checksum regex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "75ecf88252dce580404a22444fc7d626c01815debba56a7f4f536772a5ff19d3" +"checksum regex 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5bbbea44c5490a1e84357ff28b7d518b4619a159fed5d25f6c1de2d19cc42814" "checksum regex-syntax 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "8e931c58b93d86f080c734bfd2bce7dd0079ae2331235818133c8be7f422e20e" -"checksum regex-syntax 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7d707a4fa2637f2dca2ef9fd02225ec7661fe01a53623c1e6515b6916511f7a7" -"checksum regex-syntax 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05b06a75f5217880fc5e905952a42750bf44787e56a6c6d6852ed0992f5e1d54" +"checksum regex-syntax 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "747ba3b235651f6e2f67dfa8bcdcd073ddb7c243cb21c442fc12395dfcac212d" "checksum remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3488ba1b9a2084d38645c4c08276a1752dcbf2c7130d74f1569681ad5d2799c5" "checksum rustc-demangle 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "bcfe5b13211b4d78e5c2cadfebd7769197d95c639c35a50057eb4c05de811395" "checksum rusty-fork 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ea98d8d2644fd8b4946a2be90e8c6dc52b652e03079c46e134d9815062b9082d" @@ -732,13 +693,11 @@ dependencies = [ "checksum syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad" "checksum synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6" "checksum synstructure 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3a761d12e6d8dcb4dcf952a7a89b475e3a9d69e4a69307e01a470977642914bd" -"checksum tempfile 3.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "47776f63b85777d984a50ce49d6b9e58826b6a3766a449fc95bc66cd5663c15b" -"checksum termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "adc4587ead41bf016f11af03e55a624c06568b5a19db4e90fde573d805074f83" +"checksum tempfile 3.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c4b103c6d08d323b92ff42c8ce62abcd83ca8efa7fd5bf7927efefec75f58c76" +"checksum termcolor 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "722426c4a0539da2c4ffd9b419d90ad540b4cff4a053be9069c908d4d07e2836" "checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" "checksum textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c0b59b6b4b44d867f1370ef1bd91bfb262bf07bf0ae65c202ea2fbc16153b693" "checksum thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "279ef31c19ededf577bfd12dfae728040a21f635b06a24cd670ff510edd38963" -"checksum tinycdb 0.0.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1c37d5f9e60cfdf512708a4461ad1514b8c70c2647a560a67381d55ba191512d" -"checksum tinycdb-sys 0.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "699ede6e96b7a3ce84746111e875b7f6ecbd9dd23d30300594a7e7aa69f57299" "checksum ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd2be2d6639d0f8fe6cdda291ad456e23629558d466e2789d2c3e9892bda285d" "checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" "checksum unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" @@ -753,4 +712,4 @@ dependencies = [ "checksum winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "773ef9dcc5f24b7d850d0ff101e542ff24c3b090a9768e03ff889fdef41f00fd" "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -"checksum wincolor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "eeb06499a3a4d44302791052df005d5232b927ed1a9658146d842165c4de7767" +"checksum wincolor 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b9dc3aa9dcda98b5a16150c54619c1ead22e3d3a5d458778ae914be760aa981a" diff --git a/src/storage/cdb/Cargo.toml b/src/storage/cdb/Cargo.toml index 12a7680a9..bc25f84a4 100644 --- a/src/storage/cdb/Cargo.toml +++ b/src/storage/cdb/Cargo.toml @@ -1,8 +1,11 @@ +##-- cdb Cargo.toml file --## +# note: do not change the mark above # + [workspace] members = [ "cdb_rs", - "cdb_ffi", + "cdbgen", ] [profile.release] diff --git a/src/storage/cdb/cdb.h b/src/storage/cdb/cdb.h deleted file mode 100644 index 5509ddfbd..000000000 --- a/src/storage/cdb/cdb.h +++ /dev/null @@ -1,22 +0,0 @@ -#pragma once - -#include - -#include - -struct CDBHandle; - -struct CDBBString { - uint32_t len; /* string length */ - char *data; /* string data */ -}; - -struct CDBData; - -struct CDBHandle* cdb_handle_create(const char *path); -void cdb_handle_destroy(struct CDBHandle *h); - -void cdb_setup(void); -void cdb_teardown(void); - -struct bstring *cdb_get(struct CDBHandle *h, struct bstring *key, struct bstring *value); diff --git a/src/storage/cdb/cdb_ffi/CMakeLists.txt b/src/storage/cdb/cdb_ffi/CMakeLists.txt deleted file mode 100644 index 63800768a..000000000 --- a/src/storage/cdb/cdb_ffi/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -cargo_build(NAME cdb_ffi) - diff --git a/src/storage/cdb/cdb_ffi/Cargo.toml b/src/storage/cdb/cdb_ffi/Cargo.toml deleted file mode 100644 index 96735186a..000000000 --- a/src/storage/cdb/cdb_ffi/Cargo.toml +++ /dev/null @@ -1,22 +0,0 @@ -[package] -name = "cdb_ffi" -version = "0.2.0" -authors = ["Jonathan Simms "] - -[dependencies] -ccommon_rs = { path = "../../../../deps/ccommon/rust/ccommon_rs" } -cc_binding = { path = "../../../../deps/ccommon/rust/cc_binding" } - -cdb_rs = { path = "../cdb_rs" } - -log = "~0.4" -env_logger = "~0.5.10" -libc = "~0.2.42" -bytes = "~0.4" - -[lib] -name = "cdb_ffi" -crate-type = ["cdylib", "staticlib"] - -[build-dependencies] -bindgen = "0.37.0" diff --git a/src/storage/cdb/cdb_rs.h b/src/storage/cdb/cdb_rs.h new file mode 100644 index 000000000..c109c43c8 --- /dev/null +++ b/src/storage/cdb/cdb_rs.h @@ -0,0 +1,26 @@ +#pragma once + +#include + +#include + +struct cdb_handle; + +typedef enum cdb_load_method { + CDB_HEAP = 1, + CDB_MMAP = 2, +} cdb_load_method_e; + +struct cdb_handle_create_config { + struct bstring *path; + cdb_load_method_e load_method; +}; + + +struct cdb_handle *cdb_handle_create(const struct cdb_handle_create_config *cfg); +void cdb_handle_destroy(struct cdb_handle **h); + +void cdb_setup(void); +void cdb_teardown(void); + +struct bstring *cdb_get(struct cdb_handle *h, struct bstring *key, struct bstring *value); diff --git a/src/storage/cdb/cdb_rs/CMakeLists.txt b/src/storage/cdb/cdb_rs/CMakeLists.txt new file mode 100644 index 000000000..2d155c287 --- /dev/null +++ b/src/storage/cdb/cdb_rs/CMakeLists.txt @@ -0,0 +1,2 @@ +file(WRITE CMAKE_BINARY_DIR "${CMAKE_BINARY_DIR}\n") +cargo_build(NAME cdb_rs) diff --git a/src/storage/cdb/cdb_rs/Cargo.toml b/src/storage/cdb/cdb_rs/Cargo.toml index b308e5159..2f5e982fb 100644 --- a/src/storage/cdb/cdb_rs/Cargo.toml +++ b/src/storage/cdb/cdb_rs/Cargo.toml @@ -5,18 +5,26 @@ authors = ["Jonathan D. Simms "] publish = false [dependencies] +ccommon_rs = { path = "../../../../deps/ccommon/rust/ccommon_rs" } +cc_binding = { path = "../../../../deps/ccommon/rust/cc_binding" } + bytes = "~0.4" -rand = "~0.5" -memmap = "~0.6.2" -log = "~0.4" -env_logger = "~0.5.10" clap = "~2.31.0" +env_logger = "~0.5.10" +failure = "~0.1.1" +failure_derive = "~0.1.1" +libc = "~0.2.42" +log = "~0.4" +memmap = "~0.6.2" +rand = "~0.5" [dev-dependencies] tempfile = "~3" proptest = "~0.7.1" -tinycdb = "~0.0.7" [lib] name = "cdb_rs" -crate-type = ["rlib", "dylib"] +crate-type = ["rlib", "dylib", "cdylib", "staticlib"] + +[build-dependencies] +bindgen = "0.37.0" diff --git a/src/storage/cdb/cdb_rs/build.rs b/src/storage/cdb/cdb_rs/build.rs new file mode 100644 index 000000000..45f449f4b --- /dev/null +++ b/src/storage/cdb/cdb_rs/build.rs @@ -0,0 +1,54 @@ +extern crate bindgen; + +use std::env; +use std::fs; +use std::io; +use std::io::prelude::*; +use std::path::PathBuf; + +fn get_cmake_binary_dir() -> io::Result { + // this file is written by cmake on each run, updated with the location of + // the build directory. + let mut fp = fs::File::open("CMAKE_BINARY_DIR")?; + let mut buf = String::new(); + let n = fp.read_to_string(&mut buf)?; + assert!(n > 0, "file was empty"); + Ok(String::from(buf.trim_right())) +} + +fn main() { + let cwd = env::current_dir().unwrap(); + eprintln!("CWD: {}", cwd.to_str().unwrap()); + + let ccommon_include = fs::canonicalize("./../../../../deps/ccommon/include").unwrap(); + let include_path = fs::canonicalize("./..").unwrap(); + + eprintln!("ccommon_include: {}", ccommon_include.to_str().unwrap()); + eprintln!("include_path: {}", include_path.to_str().unwrap()); + + let cmake_binary_dir = match get_cmake_binary_dir() { + Ok(p) => p, + Err(err) => panic!("Failed locating the CMAKE_BINARY_DIR file: {:#?}", err), + }; + + let cbd = PathBuf::from(cmake_binary_dir); + + let mut config_h_dir = cbd.clone(); + config_h_dir.push("ccommon"); + + let bindings = bindgen::Builder::default() + .clang_args(vec![ + "-I", include_path.to_str().unwrap(), + "-I", config_h_dir.to_str().unwrap(), + "-I", ccommon_include.to_str().unwrap(), + ]) + .header("wrapper.h") + .blacklist_type("max_align_t") // https://github.com/rust-lang-nursery/rust-bindgen/issues/550 + .generate() + .expect("Unable to generate bindings"); + + let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); + bindings + .write_to_file(out_path.join("ffigen.rs")) + .expect("Couldn't write bindings!"); +} diff --git a/src/storage/cdb/cdb_rs/src/cdb/backend.rs b/src/storage/cdb/cdb_rs/src/cdb/backend.rs new file mode 100644 index 000000000..664007d83 --- /dev/null +++ b/src/storage/cdb/cdb_rs/src/cdb/backend.rs @@ -0,0 +1,63 @@ +use memmap; +use std::ops::Deref; +use super::Result; +use std::fs::File; +use std::io::{Cursor, Read}; +use std::path::Path; +use cdb::MAIN_TABLE_SIZE_BYTES; + + +pub enum Backend { + Heap(Box<[u8]>), + MMap(memmap::Mmap), +} + +impl Deref for Backend { + type Target = [u8]; + + fn deref(&self) -> &[u8] { + match self { + Backend::Heap(bx) => &bx[..], + Backend::MMap(mm) => &mm[..], + } + } +} + +impl AsRef<[u8]> for Backend { + fn as_ref(&self) -> &[u8] { &self[..] } +} + +impl Backend { + pub fn noop() -> Result { + let v = { + let mut buf = Vec::with_capacity(MAIN_TABLE_SIZE_BYTES as usize); + let mut cur = Cursor::new(buf); + super::Writer::new(&mut cur)?; + cur.into_inner() + }; + + Ok(Backend::Heap(v.into_boxed_slice())) + } + + pub fn load_path(p: &Path) -> Result { + let fp = File::open(p)?; + Backend::load(&fp) + } + + pub fn load(mut fp: &File) -> Result { + let mut buffer = Vec::new(); + fp.read_to_end(&mut buffer)?; + Ok(Backend::Heap(buffer.into_boxed_slice())) + } + + pub fn mmap_path(p: &Path) -> Result { + let fp = File::open(p)?; + Backend::mmap(&fp) + } + + pub fn mmap(fp: &File) -> Result { + Ok(Backend::MMap( + unsafe { memmap::MmapOptions::new().map(&fp)? } + )) + } +} diff --git a/src/storage/cdb/cdb_rs/src/cdb/errors.rs b/src/storage/cdb/cdb_rs/src/cdb/errors.rs index 07260588b..4e312559a 100644 --- a/src/storage/cdb/cdb_rs/src/cdb/errors.rs +++ b/src/storage/cdb/cdb_rs/src/cdb/errors.rs @@ -1,69 +1,22 @@ -use std::error; -use std::ffi::NulError; -use std::fmt; -use std::io; -use std::num::ParseIntError; -use std::str::Utf8Error; -#[derive(Debug)] -pub enum CDBError { - IOError(io::Error), - UTF8Error(::std::str::Utf8Error), - ParseError(ParseIntError), - NulError(NulError), -} +// for docs on the 'failure' crate see https://boats.gitlab.io/failure/intro.html -impl From for CDBError { - fn from(err: ParseIntError) -> CDBError { - CDBError::ParseError(err) - } -} +use std::ops::Range; -impl From for CDBError { - fn from(err: Utf8Error) -> CDBError { - CDBError::UTF8Error(err) - } -} - -impl From for CDBError { - fn from(err: io::Error) -> CDBError { - CDBError::IOError(err) - } -} - -impl From for CDBError { - fn from(err: NulError) -> CDBError { - CDBError::NulError(err) - } -} - -impl error::Error for CDBError { - fn description(&self) -> &str { - match *self { - CDBError::IOError(ref err) => err.description(), - CDBError::UTF8Error(ref err) => err.description(), - CDBError::ParseError(ref err) => err.description(), - CDBError::NulError(ref err) => err.description(), - } - } - - fn cause(&self) -> Option<&error::Error> { - match *self { - CDBError::IOError(ref err) => Some(err), - CDBError::UTF8Error(ref err) => Some(err), - CDBError::ParseError(ref err) => Some(err), - CDBError::NulError(ref err) => Some(err), - } - } +#[derive(Debug, Fail)] +pub enum CDBError { + #[fail(display = "Value too large, max_size: {}, val_size: {}", max_size, val_size)] + ValueTooLarge{max_size: usize, val_size: usize}, + + #[fail( + display = "pointer {:?} out of valid range {:?} for data segment", + ptr_val, valid_range + )] + IndexOutOfDataSegment{valid_range: Range, ptr_val: usize} } -impl fmt::Display for CDBError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match *self { - CDBError::IOError(ref err) => err.fmt(f), - CDBError::UTF8Error(ref err) => err.fmt(f), - CDBError::ParseError(ref err) => err.fmt(f), - CDBError::NulError(ref err) => err.fmt(f), - } +impl CDBError { + pub fn value_too_large(max_size: usize, val_size: usize) -> CDBError { + CDBError::ValueTooLarge{max_size, val_size} } } diff --git a/src/storage/cdb/cdb_rs/src/cdb/ffi/gen.rs b/src/storage/cdb/cdb_rs/src/cdb/ffi/gen.rs new file mode 100644 index 000000000..d1e6d3675 --- /dev/null +++ b/src/storage/cdb/cdb_rs/src/cdb/ffi/gen.rs @@ -0,0 +1,8 @@ +#![allow(unknown_lints)] +#![allow(clippy)] +#![allow(clippy_pedantic)] +#![allow(non_upper_case_globals)] +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(dead_code)] +include!(concat!(env!("OUT_DIR"), "/ffigen.rs")); diff --git a/src/storage/cdb/cdb_ffi/src/lib.rs b/src/storage/cdb/cdb_rs/src/cdb/ffi/mod.rs similarity index 62% rename from src/storage/cdb/cdb_ffi/src/lib.rs rename to src/storage/cdb/cdb_rs/src/cdb/ffi/mod.rs index f7f6fc445..a49d685e2 100644 --- a/src/storage/cdb/cdb_ffi/src/lib.rs +++ b/src/storage/cdb/cdb_rs/src/cdb/ffi/mod.rs @@ -1,73 +1,48 @@ -extern crate bytes; -extern crate cc_binding; -extern crate ccommon_rs; -extern crate cdb_rs; -extern crate env_logger; -extern crate libc; -#[macro_use] extern crate log; - use cc_binding as bind; use ccommon_rs::bstring::BStr; -use cdb_rs::{CDB, Result}; -use cdb_rs::cdb; +use cdb::{cdb_handle, Reader, Result}; +use cdb; +use env_logger; // TODO: switch to cc_log_rs use std::convert::From; use std::ffi::CStr; use std::os::raw::c_char; use std::ptr; +pub(in super) mod gen; -#[repr(C)] -pub struct CDBHandle { - inner: Box<[u8]>, -} - -impl CDBHandle { - pub unsafe fn from_raw<'a>(ptr: *mut CDBHandle) -> &'a CDBHandle { &*ptr } -} - -impl<'a> From<&'a CDBHandle> for CDB<'a> { - fn from(h: &'a CDBHandle) -> Self { - CDB::new(&h.inner) - } -} -fn mk_cdb_handler(path: &str) -> Result { +fn mk_cdb_handler(path: &str) -> Result { assert!( !path.is_empty(), "cdb file path was empty, misconfiguration?" ); debug!("mk_cdb_handler, path: {:?}", path); let inner = cdb::load_bytes_at_path(path)?; - debug!("inner: {:?}", inner); - Ok(CDBHandle { inner }) + Ok(cdb_handle::new(inner)) } fn cstr_to_string(s: *const c_char) -> Result { let ps = unsafe { CStr::from_ptr(s) }.to_str()?; - let rv = String::from(ps); - eprintln!("cstr_to_string: {:?}", rv); - - Ok(rv) + Ok(String::from(ps)) } #[no_mangle] -pub extern "C" fn cdb_handle_create(path: *const c_char) -> *mut CDBHandle { +pub extern "C" fn cdb_handle_create(path: *const c_char) -> *mut cdb_handle { assert!(!path.is_null()); match cstr_to_string(path).and_then(|s| mk_cdb_handler(&s)) { Ok(bhandle) => Box::into_raw(Box::new(bhandle)), Err(err) => { - error!("failed to create CDBHandle: {:?}", err); + error!("failed to create cdb_handle: {:?}", err); ptr::null_mut() } } } - #[no_mangle] pub unsafe extern "C" fn cdb_get( - h: *mut CDBHandle, + h: *mut cdb_handle, k: *const bind::bstring, v: *mut bind::bstring, ) -> *mut bind::bstring { @@ -76,11 +51,11 @@ pub unsafe extern "C" fn cdb_get( assert!(!v.is_null()); // TODO: don't do unwrap, be safe - let handle = CDBHandle::from_raw(h); + let handle = cdb_handle::from_raw(h); let key = BStr::from_ptr(k as *mut _); let mut val = BStr::from_ptr_mut(v); - match CDB::from(handle).get(&key, &mut val) { + match Reader::from(handle).get(&key, &mut val) { Ok(Some(n)) => { { // this provides access to the underlying struct fields @@ -101,17 +76,38 @@ pub unsafe extern "C" fn cdb_get( #[no_mangle] -pub unsafe extern "C" fn cdb_handle_destroy(handle: *mut CDBHandle) { - drop(Box::from_raw(handle)); +pub unsafe extern "C" fn cdb_handle_destroy(handle: *mut *mut cdb_handle) { + drop(Box::from_raw(*handle)); + *handle = ptr::null_mut() } #[no_mangle] pub extern "C" fn cdb_setup() { env_logger::init(); - debug!("setup cdb"); + eprintln!("setup cdb"); } #[no_mangle] pub extern "C" fn cdb_teardown() { - debug!("teardown cdb"); + eprintln!("teardown cdb"); +} + + +#[cfg(test)] +mod test { + use super::*; + use cdb::backend::Backend; + use cdb::cdb_handle; + + #[test] + fn cdb_handle_destroy_should_null_out_the_passed_ptr() { + let be = Backend::noop().unwrap(); + + let handle = Box::new(cdb_handle::from(be)); + let mut p = Box::into_raw(handle) as *mut cdb_handle; + + let pp = (&mut p) as *mut *mut cdb_handle; + unsafe { cdb_handle_destroy(pp) }; + assert!(p.is_null()); + } } diff --git a/src/storage/cdb/cdb_rs/src/cdb/input.rs b/src/storage/cdb/cdb_rs/src/cdb/input.rs index f8ea111d8..d803b2fa3 100644 --- a/src/storage/cdb/cdb_rs/src/cdb/input.rs +++ b/src/storage/cdb/cdb_rs/src/cdb/input.rs @@ -1,10 +1,11 @@ use super::KV; -use super::errors::CDBError; use bytes::*; use std::io::BufReader; use std::io::prelude::*; use std::str; +use super::Result; + struct KVSizes(usize, usize); const PLUS: u8 = 0x2b; // ASCII '+' @@ -12,19 +13,15 @@ const COMMA: u8 = 0x2c; // ASCII ',' const COLON: u8 = 0x3a; // ASCII ':' const NL: u8 = 0x0a; // ASCII '\n' -fn parse_digits(buf: &[u8]) -> Result { - str::from_utf8(&buf) - .map_err(CDBError::UTF8Error) - .and_then(|str| { - str.parse::() - .map_err(CDBError::ParseError) - }) +fn parse_digits(buf: &[u8]) -> Result { + let s = str::from_utf8(&buf)?; + s.parse::().map_err(|e| e.into()) } const ARROW_BYTES: &[u8; 2] = b"->"; // format: +1,3:a->xyz\n -fn read_begin(input: &mut BufReader) -> Result, CDBError> { +fn read_begin(input: &mut BufReader) -> Result> { let mut buf = vec![0u8; 1]; // consume a '+' @@ -38,7 +35,7 @@ fn read_begin(input: &mut BufReader) -> Result, CDBError> } } -fn read_sizes(input: &mut BufReader) -> Result { +fn read_sizes(input: &mut BufReader) -> Result { let mut buf: Vec = Vec::new(); let r = input.read_until(COMMA, &mut buf)?; @@ -58,7 +55,7 @@ fn read_sizes(input: &mut BufReader) -> Result { Ok(KVSizes(k, v)) } -fn read_kv(input: &mut BufReader, kvs: &KVSizes) -> Result { +fn read_kv(input: &mut BufReader, kvs: &KVSizes) -> Result { let KVSizes(ksize, vsize) = kvs; let mut kbytes = vec![0u8; *ksize]; @@ -88,7 +85,7 @@ fn read_kv(input: &mut BufReader, kvs: &KVSizes) -> Result(input: &mut BufReader) -> Result, CDBError> { +fn read_one_record(input: &mut BufReader) -> Result> { match read_begin(input)? { None => Ok(None), Some(_) => read_sizes(input) @@ -102,7 +99,7 @@ pub struct IterParser { } impl Iterator for IterParser { - type Item = Result; + type Item = Result; fn next(&mut self) -> Option<::Item> { match read_one_record(&mut self.buf) { @@ -127,7 +124,7 @@ mod tests { #[test] fn parser_iter() { let reader = Bytes::from("+3,4:cat->ball\n\n").into_buf().reader(); - let recs: Vec> = parse(reader).collect(); + let recs: Vec> = parse(reader).collect(); assert_eq!(recs.len(), 1); match recs[0] { diff --git a/src/storage/cdb/cdb_rs/src/cdb/mod.rs b/src/storage/cdb/cdb_rs/src/cdb/mod.rs index 751b40484..c30da5e25 100644 --- a/src/storage/cdb/cdb_rs/src/cdb/mod.rs +++ b/src/storage/cdb/cdb_rs/src/cdb/mod.rs @@ -1,27 +1,37 @@ +pub use self::backend::Backend; pub use self::errors::CDBError; - +use self::ffi::gen; use bytes::{Buf, Bytes, IntoBuf}; +use bytes::{BufMut, BytesMut}; +use cc_binding as bind; +use ccommon_rs::bstring::BString; +use failure; use std::cmp; use std::fmt; use std::fs::File; use std::io::Read; +use std::io::{Seek, SeekFrom, Write}; +use std::ops::Deref; +use std::path::Path; use std::result; +pub mod backend; pub mod errors; +pub mod ffi; pub mod input; pub mod storage; pub const STARTING_HASH: u32 = 5381; const MAIN_TABLE_SIZE: usize = 256; -const MAIN_TABLE_SIZE_BYTES: usize = 2048; -const END_TABLE_ENTRY_SIZE: usize = 8; -const DATA_HEADER_SIZE: usize = 8; +const MAIN_TABLE_SIZE_BYTES: u32 = 2048; +const END_TABLE_ENTRY_SIZE: u32 = 8; +const DATA_HEADER_SIZE: u32 = 8; +const INDEX_ENTRY_SIZE: usize = 8; -pub type Result = result::Result; +pub type Result = result::Result; // idea from https://raw.githubusercontent.com/jothan/cordoba/master/src/lib.rs -#[derive(Copy, Clone, Eq, PartialEq)] -#[repr(C)] +#[derive(Copy, Clone, Eq, PartialEq, Default)] struct CDBHash(u32); impl CDBHash { @@ -32,22 +42,30 @@ impl CDBHash { // wrapping here is explicitly for allowing overflow semantics: // // Operations like + on u32 values is intended to never overflow, - // and in some debug configurations overflow is detected and results in a panic. - // While most arithmetic falls into this category, some code explicitly expects - // and relies upon modular arithmetic (e.g., hashing) + // and in some debug configurations overflow is detected and + // results in a panic. While most arithmetic falls into this + // category, some code explicitly expects and relies upon + // modular arithmetic (e.g., hashing) // h = h.wrapping_shl(5).wrapping_add(h) ^ u32::from(*b) } CDBHash(h) } + #[inline] fn table(self) -> usize { self.0 as usize % MAIN_TABLE_SIZE } + #[inline] fn slot(self, num_ents: usize) -> usize { (self.0 as usize >> 8) % num_ents } + + #[inline] + fn inner(self) -> u32 { + self.0 + } } impl fmt::Debug for CDBHash { @@ -57,22 +75,23 @@ impl fmt::Debug for CDBHash { } impl<'a> From<&'a CDBHash> for usize { + #[inline] fn from(h: &'a CDBHash) -> Self { h.0 as usize } } impl<'a> From<&'a CDBHash> for u32 { + #[inline] fn from(h: &'a CDBHash) -> Self { h.0 } } #[derive(Copy, Clone)] -#[repr(C)] struct Bucket { - ptr: usize, - num_ents: usize, + ptr: u32, + num_ents: u32, } impl fmt::Debug for Bucket { @@ -88,48 +107,127 @@ impl fmt::Debug for Bucket { impl Bucket { // returns the offset into the db of entry n of this bucket. // panics if n >= num_ents - fn entry_n_pos(&self, n: usize) -> IndexEntryPos { + #[inline] + fn entry_n_pos(self, n: u32) -> IndexEntryPos { assert!(n < self.num_ents); IndexEntryPos(self.ptr + (n * END_TABLE_ENTRY_SIZE)) } } #[derive(Copy, Clone, Debug, Eq, PartialEq)] -#[repr(C)] -struct IndexEntryPos(usize); +struct IndexEntryPos(u32); -impl From for usize { +impl From for u32 { + #[inline] fn from(n: IndexEntryPos) -> Self { n.0 } } +impl From for usize { + #[inline] + fn from(n: IndexEntryPos) -> Self { + n.0 as usize + } +} + #[derive(Clone, Debug)] -#[repr(C)] pub struct KV { pub k: Bytes, pub v: Bytes, } #[derive(Clone, Debug)] -#[repr(C)] pub struct KVRef<'a> { pub k: &'a [u8], pub v: &'a [u8], } -#[repr(C)] +#[derive(Copy, Clone, Default)] struct IndexEntry { - hash: CDBHash, // the hash of the stored key - ptr: usize, // pointer to the absolute position of the data in the db + hash: CDBHash, + // the hash of the stored key + ptr: u32, // pointer to the absolute position of the data in the db } -#[derive(Debug)] -#[repr(C)] -pub struct CDB<'a> { - data: &'a [u8], +enum LoadMethod { + HEAP = 1, + MMAP = 2, +} + +pub struct CDBHandleConfig { + bpath: BString, + load_method: LoadMethod, +} + +impl CDBHandleConfig { + pub unsafe fn from_raw(ptr: *const gen::cdb_handle_create_config) -> CDBHandleConfig { + assert!(!ptr.is_null()); + + let cfg = &*ptr; + // this looks nasty, the only reason we have to do this is that rust + // sees the type of gen::cdb_handle_create_config.path as a cdb::ffi::gen::bstring + // which is the same as cc_binding::bstring, so we just coerce one into the other + // to make the compiler happy + let bpath = BString::from_raw(&mut *(cfg.path) as *mut _ as *mut bind::bstring); + + let load_method = match cfg.load_method { + gen::cdb_load_method_CDB_HEAP => LoadMethod::HEAP, + gen::cdb_load_method_CDB_MMAP => LoadMethod::MMAP, + _ => unreachable!(), + }; + + CDBHandleConfig { bpath, load_method } + } + + pub fn path(&self) -> Result<&Path> { + ::std::str::from_utf8(&*self.bpath) + .map(Path::new) + .map_err(|e| e.into()) + } + + pub fn into_cdb_handle(self) -> Result { + match self.load_method { + LoadMethod::HEAP => self.path() + .and_then(backend::Backend::load_path) + .map(cdb_handle::from), + LoadMethod::MMAP => self.path() + .and_then(backend::Backend::mmap_path) + .map(cdb_handle::from), + } + } } +// this struct crosses the FFI boundary (as a pointer), so we give it +// a snake_case name so it matches the style convention on the C side. +#[allow(non_camel_case_types)] +pub struct cdb_handle(Backend); + +impl cdb_handle { + pub unsafe fn from_raw<'a>(ptr: *mut cdb_handle) -> &'a cdb_handle { + &*(ptr as *mut _) + } + + pub fn new(b: Box<[u8]>) -> cdb_handle { + cdb_handle(Backend::Heap(b)) + } +} + +impl From for cdb_handle { + fn from(be: Backend) -> Self { + cdb_handle(be) + } +} + +impl<'a> From<&'a cdb_handle> for Reader<'a> { + fn from(h: &'a cdb_handle) -> Self { + Reader(&h.0[..]) + } +} + +#[derive(Debug)] +pub struct Reader<'a>(&'a [u8]); + pub fn load_bytes_at_path(path: &str) -> Result> { let mut f = File::open(path)?; let mut buffer = Vec::with_capacity(f.metadata()?.len() as usize); @@ -137,26 +235,30 @@ pub fn load_bytes_at_path(path: &str) -> Result> { Ok(buffer.into_boxed_slice()) } -impl<'a> CDB<'a> { - pub fn new(b: &[u8]) -> CDB { - CDB { data: b } +impl<'a> Reader<'a> { + pub fn new>(r: &'a T) -> Reader<'a> { + Reader(r.as_ref()) } + // TODO: perform basic sanity checks of data (gee i wish there was a checksum + // somwhere in this data format HINT HINT) + // + // * does the secondary index fall safely in the correct index range? + // * do data pointers point at the data segment? + #[inline] fn bucket_at(&self, idx: usize) -> Result { assert!(idx < MAIN_TABLE_SIZE); let off = 8 * idx; - let slice = self.data[off..(off + 8)].as_ref(); - let b = slice.into_buf(); + let slice = self[off..(off + 8)].as_ref(); + let mut b = slice.into_buf(); assert_eq!(slice.len(), 8); trace!("bucket_at idx: {}, got buf: {:?}", idx, b); - let mut buf = b.into_buf(); - - let ptr = buf.get_u32_le() as usize; - let num_ents = buf.get_u32_le() as usize; + let ptr = b.get_u32_le(); + let num_ents = b.get_u32_le(); Ok(Bucket { ptr, num_ents }) } @@ -166,34 +268,35 @@ impl<'a> CDB<'a> { fn index_entry_at(&self, pos: IndexEntryPos) -> Result { let pos: usize = pos.into(); - if pos < MAIN_TABLE_SIZE_BYTES { + if (pos as u32) < MAIN_TABLE_SIZE_BYTES { panic!("position {:?} was in the main table!", pos) } - let mut b = self.data[pos..(pos + 8)].into_buf(); + let mut b = self[pos..(pos + 8)].into_buf(); let hash = CDBHash(b.get_u32_le()); - let ptr = b.get_u32_le() as usize; + let ptr = b.get_u32_le(); Ok(IndexEntry { hash, ptr }) } #[inline] - fn get_kv_ref(&self, ie: &IndexEntry) -> Result> { - let b = self.data[ie.ptr..(ie.ptr + DATA_HEADER_SIZE)].as_ref(); + fn get_kv_ref(&'a self, ie: IndexEntry) -> Result> { + let p = ie.ptr as usize; + let b = self[p..(p + DATA_HEADER_SIZE as usize)].as_ref(); let ksize = b[..4].into_buf().get_u32_le() as usize; let vsize = b[4..].into_buf().get_u32_le() as usize; - let kstart = ie.ptr + DATA_HEADER_SIZE; - let vstart = kstart + ksize; + let kstart = p + DATA_HEADER_SIZE as usize; + let vstart = (kstart + ksize) as usize; - let k = &self.data[kstart..(kstart + ksize)]; - let v = &self.data[vstart..(vstart + vsize)]; + let k = &self[kstart..(kstart + ksize)]; + let v = &self[vstart..(vstart + vsize)]; Ok(KVRef { k, v }) } - pub fn get(&self, key: &[u8], buf: &mut[u8]) -> Result> { + pub fn get(&self, key: &[u8], buf: &mut [u8]) -> Result> { let hash = CDBHash::new(key); let bucket = self.bucket_at(hash.table())?; @@ -202,18 +305,17 @@ impl<'a> CDB<'a> { return Ok(None); } - let slot = hash.slot(bucket.num_ents); + let slot = hash.slot(bucket.num_ents as usize); for x in 0..bucket.num_ents { - let index_entry_pos = bucket.entry_n_pos((x + slot) % bucket.num_ents); + let index_entry_pos = bucket.entry_n_pos((x + slot as u32) % bucket.num_ents); let idx_ent = self.index_entry_at(index_entry_pos)?; if idx_ent.ptr == 0 { return Ok(None); } else if idx_ent.hash == hash { - let kv = self.get_kv_ref(&idx_ent)?; - // TODO: this is incorrect handling of the buffer! shit! + let kv = self.get_kv_ref(idx_ent)?; if &kv.k[..] == key { return Ok(Some(copy_slice(buf, kv.v))); } else { @@ -226,6 +328,14 @@ impl<'a> CDB<'a> { } } +impl<'a> Deref for Reader<'a> { + type Target = [u8]; + + fn deref(&self) -> &[u8] { + self.0 + } +} + #[inline] fn copy_slice(dst: &mut [u8], src: &[u8]) -> usize { let n = cmp::min(dst.len(), src.len()); @@ -235,60 +345,26 @@ fn copy_slice(dst: &mut [u8], src: &[u8]) -> usize { #[cfg(test)] mod tests { - use std::collections::hash_set; - use std::fs::remove_file; - use std::path::PathBuf; use super::*; use tempfile::NamedTempFile; - use tinycdb::Cdb as TCDB; - - fn create_temp_cdb<'a>(kvs: &Vec<(String, String)>) -> Result> { - let path: PathBuf; - - { - let ntf = NamedTempFile::new()?; - remove_file(ntf.path())?; - path = ntf.path().to_path_buf(); - } - - let mut dupcheck = hash_set::HashSet::new(); - - TCDB::new(path.as_ref(), |c| { - let ys = kvs.to_owned(); - for (k, v) in ys { - let kk = k.clone(); - let vv = v.clone(); - - if !dupcheck.contains(&k) { - dupcheck.insert(k); - c.add(kk.as_ref(), vv.as_ref()).unwrap(); - } - } - }).unwrap(); - - load_bytes_at_path(path.to_str().unwrap()) - } - #[test] - fn round_trip_test() { - let kvs: Vec<(String, String)> = vec![ + fn kvs() -> Vec<(String, String)> { + vec![ ("abc", "def"), ("pink", "red"), ("apple", "grape"), ("q", "burp"), ].iter() - .map(|(k,v)| (k.to_string(), v.to_string())) - .collect(); - - let data = create_temp_cdb(&kvs).unwrap(); - - let cdb = CDB { data: &data }; + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() + } - for (k, v) in kvs { + fn validate(pairs: &Vec<(String, String)>, cdb: &Reader) -> Result<()> { + for (k, v) in pairs { let mut buf = Vec::new(); buf.resize(10, 0u8); - + let n = cdb.get(k.as_bytes(), &mut buf[..]).unwrap().unwrap(); assert_eq!(n, v.len()); assert_eq!(&buf[0..n], v.as_bytes()) @@ -299,5 +375,171 @@ mod tests { let r = cdb.get("1233".as_bytes(), &mut buf[..]).unwrap(); assert!(r.is_none()); + Ok(()) + } + + fn load_and_validate_cdb<'a, F, T>(kvs: &Vec<(String, String)>, f: F) -> Result<()> + where + F: FnOnce(&mut NamedTempFile) -> Result, + T: AsRef<[u8]>, + { + let mut ntf = NamedTempFile::new()?; + + { + let mut w = Writer::new(ntf.as_file_mut())?; + for kv in kvs { + let (k, v) = kv.clone(); + w.put(&k.into_bytes(), &v.into_bytes())?; + } + } + + let data = f(&mut ntf)?; + let cdb = Reader::new(&data); + validate(&kvs, &cdb) + } + + #[test] + fn round_trip_boxed_slice_test() { + let pairs = kvs(); + + load_and_validate_cdb(&pairs, |ntf| { + let mut buf = Vec::new(); + ntf.read_to_end(&mut buf)?; + Ok(buf.into_boxed_slice()) + }).unwrap() + } + + #[test] + fn round_trip_heap_backend() { + let pairs = kvs(); + + load_and_validate_cdb(&pairs, |ntf| { + Backend::load(ntf.as_file_mut()) + }).unwrap() + } + + #[test] + fn round_trip_mmap_backend() { + let pairs = kvs(); + + load_and_validate_cdb(&pairs, |ntf| { + Backend::mmap(ntf.as_file_mut()) + }).unwrap() + } +} + +fn ready_buf(size: usize) -> BytesMut { + let mut b = BytesMut::with_capacity(size); + b.reserve(size); + b +} + +pub struct Writer<'a, F> + where + F: Write + Seek + 'a, +{ + file: &'a mut F, + index: Vec>, +} + +impl<'a, F> Writer<'a, F> + where + F: Write + Seek + 'a, +{ + pub fn new(file: &'a mut F) -> Result> { + file.seek(SeekFrom::Start(0))?; + file.write_all(&[0u8; MAIN_TABLE_SIZE_BYTES as usize])?; + + Ok(Writer { + file, + index: vec![vec![IndexEntry::default()]; 256], + }) + } + + fn seek(&mut self, sf: SeekFrom) -> Result { + self.file.seek(sf).map(|n| n as u32).map_err(|e| e.into()) + } + + pub fn put(&mut self, key: &[u8], value: &[u8]) -> Result<()> { + let ptr = self.seek(SeekFrom::Current(0))?; + let mut buf = ready_buf(INDEX_ENTRY_SIZE + key.len() + value.len()); + + buf.put_u32_le(key.len() as u32); + buf.put_u32_le(value.len() as u32); + buf.extend_from_slice(key); + buf.extend_from_slice(value); + + self.file.write_all(&buf[..])?; + + let hash = CDBHash::new(key); + self.index[hash.table() as usize].push(IndexEntry { hash, ptr }); + Ok(()) + } + + fn finalize(&mut self) -> Result<()> { + let mut buckets: Vec = Vec::with_capacity(256); + self.file.seek(SeekFrom::End(0))?; + + let idx = self.index.clone(); + + for tbl in idx { + let length = (tbl.len() << 1) as u32; + let mut ordered: Vec = vec![IndexEntry::default(); length as usize]; + for idx_ent in tbl { + let slot = idx_ent.hash.slot(length as usize); + for i in 0..length { + let j = (i + slot as u32) % length; + if ordered[j as usize].ptr == 0 { + ordered[j as usize] = idx_ent; + break; + } + } + } + + // move to EOF and write out the secondary index entries, constructing the + // primary table as we go ('buckets') + // + buckets.push(Bucket { + ptr: self.seek(SeekFrom::End(0))?, + num_ents: length, + }); + + let mut buf = ready_buf((ordered.len() * 8) as usize); + + for idx_ent in ordered { + buf.put_u32_le(idx_ent.hash.inner()); + buf.put_u32_le(idx_ent.ptr); + } + + self.file.write_all(&buf[..])?; + } + + // now write the buckets + // + self.file.seek(SeekFrom::Start(0))?; + { + let mut buf = ready_buf(2048); + + for bkt in buckets { + buf.put_u32_le(bkt.ptr); + buf.put_u32_le(bkt.num_ents); + } + + self.file.write_all(&buf[..])?; + } + + // start at BOF + self.file.seek(SeekFrom::Start(0))?; + + Ok(()) + } +} + +impl<'a, F> Drop for Writer<'a, F> + where + F: Write + Seek + 'a, +{ + fn drop(&mut self) { + self.finalize().unwrap(); } } diff --git a/src/storage/cdb/cdb_rs/src/cdb/storage.rs b/src/storage/cdb/cdb_rs/src/cdb/storage.rs index 6b1b20bc7..a943fe573 100644 --- a/src/storage/cdb/cdb_rs/src/cdb/storage.rs +++ b/src/storage/cdb/cdb_rs/src/cdb/storage.rs @@ -9,7 +9,6 @@ use std::os::unix::fs::FileExt; use std::sync::Arc; #[derive(Debug)] -#[repr(C, u8)] pub enum SliceFactory { HeapStorage(HeapWrap), MmapStorage(MMapWrap), diff --git a/src/storage/cdb/cdb_rs/src/lib.rs b/src/storage/cdb/cdb_rs/src/lib.rs index 446f7cfd4..92079c852 100644 --- a/src/storage/cdb/cdb_rs/src/lib.rs +++ b/src/storage/cdb/cdb_rs/src/lib.rs @@ -1,13 +1,20 @@ extern crate bytes; -extern crate memmap; - -#[macro_use] extern crate log; -extern crate env_logger; +extern crate cc_binding; +extern crate ccommon_rs; extern crate clap; +extern crate env_logger; +extern crate failure; +#[macro_use] +extern crate failure_derive; +extern crate libc; +#[macro_use] +extern crate log; +extern crate memmap; // dev dependencies -#[cfg(test)] extern crate tempfile; -#[cfg(test)] extern crate tinycdb; +#[cfg(test)] +extern crate tempfile; pub mod cdb; -pub use cdb::{CDB, Result, CDBError}; +pub use cdb::{cdb_handle, CDBError, Reader, Result, Writer}; +pub use memmap::Mmap; diff --git a/src/storage/cdb/cdb_rs/wrapper.h b/src/storage/cdb/cdb_rs/wrapper.h new file mode 100644 index 000000000..7ca474e74 --- /dev/null +++ b/src/storage/cdb/cdb_rs/wrapper.h @@ -0,0 +1 @@ +#include diff --git a/src/storage/cdb/cdbgen/Cargo.toml b/src/storage/cdb/cdbgen/Cargo.toml new file mode 100644 index 000000000..9c5e0793f --- /dev/null +++ b/src/storage/cdb/cdbgen/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "cdbgen" +version = "0.1.0" +authors = ["Jonathan D. Simms "] + +[dependencies] +clap = "~2.31" +tempfile = "~3.0" +cdb_rs = { path = "../cdb_rs" } + +[lib] +name = "cdbgen" +path = "src/lib.rs" + +[[bin]] +name = "cdbgen" +path = "src/main.rs" + diff --git a/src/storage/cdb/cdbgen/src/gen/mod.rs b/src/storage/cdb/cdbgen/src/gen/mod.rs new file mode 100644 index 000000000..fb757d948 --- /dev/null +++ b/src/storage/cdb/cdbgen/src/gen/mod.rs @@ -0,0 +1,87 @@ + +use std::env; +use std::ops::Range; +use std::path::PathBuf; +use tempfile::NamedTempFile; +use std::io::{BufWriter}; + +use cdb_rs::*; + + +fn parent_dir(pb: &PathBuf) -> Result { + if pb.is_relative() { + let mut cwd = env::current_dir()?; + cwd.push(pb); + return parent_dir(&cwd); + } + + Ok(pb.parent().unwrap().to_path_buf()) +} + +const ASCII: Range = 32u8..127u8; + +pub fn create(path: &PathBuf) -> Result<()> { + let mut bw = BufWriter::new( + NamedTempFile::new_in(parent_dir(path)?)?); + + { + let mut w = Writer::new(&mut bw)?; + + for x in ASCII { + let k = [x]; + let v = [x]; + w.put(&k, &v)?; + + for y in ASCII { + let k = [x, y]; + let v = [x, y]; + w.put(&k, &v)?; + } + } + } + + let tf = bw.into_inner()?; + tf.as_file().sync_all()?; + tf.persist(path) + .map(|_| ()) + .map_err(|e| e.into()) +} + + +#[cfg(test)] +mod test { + use super::*; + use tempfile; + use std::fs::File; + use std::io::prelude::*; + + #[test] + fn create_and_read() { + let tempdir = tempfile::TempDir::new().unwrap(); + + let mut pb = tempdir.path().to_path_buf(); + pb.push("db.cdb"); + create(&pb).unwrap(); + + let data = { + let mut f = File::open(pb).unwrap(); + assert!(f.metadata().unwrap().len() > 2048); + let mut b = Vec::new(); + + let sz = f.read_to_end(&mut b).unwrap(); + assert_eq!(f.metadata().unwrap().len() as usize, sz); + b.into_boxed_slice() + }; + + let reader = Reader::new(&data); + + for x in ASCII { + let mut buf = vec![0u8; 2]; + let key = vec![x, x]; + + let sz = reader.get(&key, &mut buf[..]).unwrap().unwrap(); + assert_eq!(sz, key.len()); + assert_eq!(&buf[0..key.len()], &key[..]); + } + } +} diff --git a/src/storage/cdb/cdbgen/src/lib.rs b/src/storage/cdb/cdbgen/src/lib.rs new file mode 100644 index 000000000..b7f103804 --- /dev/null +++ b/src/storage/cdb/cdbgen/src/lib.rs @@ -0,0 +1,5 @@ +extern crate cdb_rs; +extern crate tempfile; + +mod gen; +pub use gen::create; diff --git a/src/storage/cdb/cdbgen/src/main.rs b/src/storage/cdb/cdbgen/src/main.rs new file mode 100644 index 000000000..470948e62 --- /dev/null +++ b/src/storage/cdb/cdbgen/src/main.rs @@ -0,0 +1,24 @@ +extern crate cdb_rs; +extern crate cdbgen; +extern crate clap; +extern crate tempfile; + +use clap::{App, Arg}; +use std::path::PathBuf; + +fn main() { + let matches = App::new("cdbgen") + .version("0.1.0") + .author("Jonathan Simms") + .about("Creates a cdb file with n-byte keys for testing") + .arg( + Arg::with_name("OUTPUT") + .help("path to write cdb to") + .required(true) + .index(1), + ) + .get_matches(); + + let output = PathBuf::from(matches.value_of("OUTPUT").unwrap()); + cdbgen::create(&output).unwrap(); +} diff --git a/src/storage/cdb/scripts/build-cargo-config.sh b/src/storage/cdb/scripts/build-cargo-config.sh new file mode 100755 index 000000000..0628efd8e --- /dev/null +++ b/src/storage/cdb/scripts/build-cargo-config.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# utility script run by cmake for writing a .cargo/config that points +# to the common 'target' directory under the CMAKE_BINARY_DIR. This +# allows automated tools (such as the intellij rust plugin or vcode +# rust integration) to share output and avoid recompiling between +# the command line and the IDE. +# +# it is assumed that this script is run with the CWD being the +# place where the .cargo dir should be created. + +set -euo pipefail +IFS=$'\n\t' + +die() { echo "fatal: $*" >&2; exit 1; } + +if [[ -z "${CMAKE_BINARY_DIR}" ]]; then + die "CMAKE_BINARY_DIR must be set!" +fi + +mkdir -p .cargo + +cleanup() { + [[ -n "${TEMPFILE:-}" ]] && rm -rf "$TEMPFILE" +} +trap cleanup EXIT + +TEMPFILE="$(mktemp '.cargo/config.XXXXXXXX')" || die "could not create tempfile" + +cat > "$TEMPFILE" <