From 26e34d15022f64a689e93cef804281799ed5818a Mon Sep 17 00:00:00 2001 From: "Jonathan D. Simms" Date: Wed, 20 Jun 2018 19:36:57 +0000 Subject: [PATCH 01/24] commit cdb.rs at 95e9de11378c06728c36cf84ed0cb0293820c1a5 to src/storage/cdb --- .gitignore | 2 + CMakeLists.txt | 2 + deps/ccommon/CMakeLists.txt | 1 + src/storage/CMakeLists.txt | 1 + src/storage/cdb/.gitignore | 7 + src/storage/cdb/CMakeLists.txt | 20 + src/storage/cdb/Cargo.lock | 556 +++++++++++++++++++++ src/storage/cdb/Cargo.toml | 38 ++ src/storage/cdb/Makefile | 14 + src/storage/cdb/scripts/mkcdb.sh | 22 + src/storage/cdb/src/bin/generatecdb.rs | 56 +++ src/storage/cdb/src/cdb/cdb_c.rs | 40 ++ src/storage/cdb/src/cdb/errors.rs | 67 +++ src/storage/cdb/src/cdb/input.rs | 156 ++++++ src/storage/cdb/src/cdb/mod.rs | 436 ++++++++++++++++ src/storage/cdb/src/cdb/randoread.rs | 113 +++++ src/storage/cdb/src/cdb/storage.rs | 235 +++++++++ src/storage/cdb/src/cdb/writer.rs | 3 + src/storage/cdb/src/lib.rs | 19 + src/storage/cdb/src/main.rs | 102 ++++ test/data_structure/bitmap/CMakeLists.txt | 1 + test/data_structure/ziplist/CMakeLists.txt | 2 +- test/hotkey/kc_map/CMakeLists.txt | 2 +- test/hotkey/key_window/CMakeLists.txt | 2 +- test/protocol/admin/CMakeLists.txt | 2 +- test/protocol/data/memcache/CMakeLists.txt | 2 +- test/protocol/data/redis/CMakeLists.txt | 2 +- test/storage/cuckoo/CMakeLists.txt | 1 + test/storage/slab/CMakeLists.txt | 1 + 29 files changed, 1899 insertions(+), 6 deletions(-) create mode 100644 src/storage/cdb/.gitignore create mode 100644 src/storage/cdb/CMakeLists.txt create mode 100644 src/storage/cdb/Cargo.lock create mode 100644 src/storage/cdb/Cargo.toml create mode 100644 src/storage/cdb/Makefile create mode 100755 src/storage/cdb/scripts/mkcdb.sh create mode 100644 src/storage/cdb/src/bin/generatecdb.rs create mode 100644 src/storage/cdb/src/cdb/cdb_c.rs create mode 100644 src/storage/cdb/src/cdb/errors.rs create mode 100644 src/storage/cdb/src/cdb/input.rs create mode 100644 src/storage/cdb/src/cdb/mod.rs create mode 100644 src/storage/cdb/src/cdb/randoread.rs create mode 100644 src/storage/cdb/src/cdb/storage.rs create mode 100644 src/storage/cdb/src/cdb/writer.rs create mode 100644 src/storage/cdb/src/lib.rs create mode 100644 src/storage/cdb/src/main.rs diff --git a/.gitignore b/.gitignore index ace9dc100..0f32ffdb6 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,5 @@ lcov # python compiled files *.pyc + +cdb.rs diff --git a/CMakeLists.txt b/CMakeLists.txt index 648bfc9b6..3f2bdb83b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -118,6 +118,8 @@ include_directories(${include_directories} "${CCOMMON_SOURCE_DIR}/include" "${PROJECT_SOURCE_DIR}/src") +add_subdirectory("cdb.rs") + # server & (cli) client add_subdirectory(src) diff --git a/deps/ccommon/CMakeLists.txt b/deps/ccommon/CMakeLists.txt index 1dfce90f9..b8fb9e764 100644 --- a/deps/ccommon/CMakeLists.txt +++ b/deps/ccommon/CMakeLists.txt @@ -93,6 +93,7 @@ set(CMAKE_MACOSX_RPATH 1) set(CFLAGS_LIST "-std=c11 " "-ggdb3 -O2 " + "-lsubunit " "-Wall " "-Wmissing-prototypes -Wmissing-declarations -Wredundant-decls " "-Wunused-function -Wunused-value -Wunused-variable " diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index c06eede35..e157b1eb8 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -1,2 +1,3 @@ +add_subdirectory(cdb) add_subdirectory(cuckoo) add_subdirectory(slab) diff --git a/src/storage/cdb/.gitignore b/src/storage/cdb/.gitignore new file mode 100644 index 000000000..ab4c6da5b --- /dev/null +++ b/src/storage/cdb/.gitignore @@ -0,0 +1,7 @@ + +/target +**/*.rs.bk +.vscode/ +*.cdb +perf.data* +*.cdb* diff --git a/src/storage/cdb/CMakeLists.txt b/src/storage/cdb/CMakeLists.txt new file mode 100644 index 000000000..a57d1eba7 --- /dev/null +++ b/src/storage/cdb/CMakeLists.txt @@ -0,0 +1,20 @@ +if (CMAKE_BUILD_TYPE STREQUAL "Debug") + set(CARGO_CMD cargo build) + set(TARGET_DIR "debug") +else () + set(CARGO_CMD cargo build --release) + set(TARGET_DIR "release") +endif () + +set(CDB_RS_SO "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_DIR}/libcdb_rs.so") + +add_custom_target(cdb_rs ALL + COMMENT "Compiling cdb module" + COMMAND CARGO_TARGET_DIR=${CMAKE_CURRENT_BINARY_DIR} ${CARGO_CMD} + COMMAND cp ${CDB_RS_SO} ${CMAKE_CURRENT_BINARY_DIR} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) +set_target_properties(cdb_rs PROPERTIES LOCATION ${CMAKE_CURRENT_BINARY_DIR}) + +add_test(NAME cdb_rs + COMMAND cargo test + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/src/storage/cdb/Cargo.lock b/src/storage/cdb/Cargo.lock new file mode 100644 index 000000000..90b98ce64 --- /dev/null +++ b/src/storage/cdb/Cargo.lock @@ -0,0 +1,556 @@ +[[package]] +name = "aho-corasick" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "atty" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bit-set" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bit-vec 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bit-vec" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "bitflags" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "byteorder" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "bytes" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "cdb_rs" +version = "0.1.0" +dependencies = [ + "bytes 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)", + "itertools 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", + "proptest 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rust-crypto 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", + "tempfile 3.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tinycdb 0.0.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "cfg-if" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "clap" +version = "2.31.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "either" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "env_logger" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", + "humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "fnv" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "gcc" +version = "0.3.54" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "humantime" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "iovec" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "itertools" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "either 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "lazy_static" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libc" +version = "0.2.42" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "log" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "memchr" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "memmap" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num-traits" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "proptest" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bit-set 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", + "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rusty-fork 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tempfile 3.0.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "quick-error" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "rand" +version = "0.3.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_core" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "redox_syscall" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "redox_termios" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "regex" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", + "thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "regex-syntax" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "regex-syntax" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "remove_dir_all" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rust-crypto" +version = "0.2.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "gcc 0.3.54 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rustc-serialize" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "rusty-fork" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tempfile 3.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "wait-timeout 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "strsim" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "tempfile" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "termcolor" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "wincolor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "termion" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "textwrap" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "thread_local" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "time" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tinycdb" +version = "0.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "tinycdb-sys 0.0.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tinycdb-sys" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ucd-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unicode-width" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unreachable" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "utf8-ranges" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "vec_map" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "wait-timeout" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "wincolor" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[metadata] +"checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4" +"checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +"checksum atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "2fc4a1aa4c24c0718a250f0681885c1af91419d242f29eb8f2ab28502d80dbd1" +"checksum bit-set 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6f1efcc46c18245a69c38fcc5cc650f16d3a59d034f3106e9ed63748f695730a" +"checksum bit-vec 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4440d5cb623bb7390ae27fec0bb6c61111969860f8e3ae198bfa0663645e67cf" +"checksum bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d0c54bb8f454c567f21197eefcdbf5679d0bd99f2ddbe52e84c77061952e6789" +"checksum byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "74c0b906e9446b0a2e4f760cdb3fa4b2c48cdc6db8766a845c54b6ff063fd2e9" +"checksum bytes 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7dd32989a66957d3f0cba6588f15d4281a733f4e9ffc43fcd2385f57d3bf99ff" +"checksum cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "405216fd8fe65f718daa7102ea808a946b6ce40c742998fbfd3463645552de18" +"checksum clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f0f16b89cbb9ee36d87483dc939fe9f1e13c05898d56d7b230a0d4dff033a536" +"checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +"checksum either 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3be565ca5c557d7f59e7cfcf1844f9e3033650c929c6566f511e8005f205c1d0" +"checksum env_logger 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)" = "0e6e40ebb0e66918a37b38c7acab4e10d299e0463fe2af5d29b9cc86710cfd2a" +"checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" +"checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +"checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" +"checksum gcc 0.3.54 (registry+https://github.com/rust-lang/crates.io-index)" = "5e33ec290da0d127825013597dbdfc28bee4964690c7ce1166cbc2a7bd08b1bb" +"checksum humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0484fda3e7007f2a4a0d9c3a703ca38c71c54c55602ce4660c419fd32e188c9e" +"checksum iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dbe6e417e7d0975db6512b90796e8ce223145ac4e33c377e4a42882a0e88bb08" +"checksum itertools 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "d3f2be4da1690a039e9ae5fd575f706a63ad5a2120f161b1d653c9da3930dd21" +"checksum lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e6412c5e2ad9584b0b8e979393122026cdd6d2a80b933f890dcd694ddbe73739" +"checksum libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" = "b685088df2b950fccadf07a7187c8ef846a959c142338a48f9dc0b94517eb5f1" +"checksum log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6fddaa003a65722a7fb9e26b0ce95921fe4ba590542ced664d8ce2fa26f9f3ac" +"checksum memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "796fba70e76612589ed2ce7f45282f5af869e0fdd7cc6199fa1aa1f1d591ba9d" +"checksum memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e2ffa2c986de11a9df78620c01eeaaf27d94d3ff02bf81bfcca953102dd0c6ff" +"checksum num-traits 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "775393e285254d2f5004596d69bb8bc1149754570dcc08cf30cabeba67955e28" +"checksum proptest 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2b21129762855f6e5d39628cab74d9040d63bc91edc343112d22582c9bb87a9c" +"checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0" +"checksum rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)" = "15a732abf9d20f0ad8eeb6f909bf6868722d9a06e1e50802b6a70351f40b4eb1" +"checksum rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "eba5f8cb59cc50ed56be8880a5c7b496bfd9bd26394e176bc67884094145c2c5" +"checksum rand 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "482c45f965103f2433002a0c4d908599f38d1b8c1375e66e801a24c1c6cadc03" +"checksum rand_core 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "edecf0f94da5551fc9b492093e30b041a891657db7940ee221f9d2f66e82eef2" +"checksum redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "c214e91d3ecf43e9a4e41e578973adeb14b474f2bee858742d127af75a0112b1" +"checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" +"checksum regex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "75ecf88252dce580404a22444fc7d626c01815debba56a7f4f536772a5ff19d3" +"checksum regex-syntax 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "8e931c58b93d86f080c734bfd2bce7dd0079ae2331235818133c8be7f422e20e" +"checksum regex-syntax 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05b06a75f5217880fc5e905952a42750bf44787e56a6c6d6852ed0992f5e1d54" +"checksum remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3488ba1b9a2084d38645c4c08276a1752dcbf2c7130d74f1569681ad5d2799c5" +"checksum rust-crypto 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)" = "f76d05d3993fd5f4af9434e8e436db163a12a9d40e1a58a726f27a01dfd12a2a" +"checksum rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)" = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda" +"checksum rusty-fork 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ea98d8d2644fd8b4946a2be90e8c6dc52b652e03079c46e134d9815062b9082d" +"checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550" +"checksum tempfile 3.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "47776f63b85777d984a50ce49d6b9e58826b6a3766a449fc95bc66cd5663c15b" +"checksum termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "adc4587ead41bf016f11af03e55a624c06568b5a19db4e90fde573d805074f83" +"checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" +"checksum textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c0b59b6b4b44d867f1370ef1bd91bfb262bf07bf0ae65c202ea2fbc16153b693" +"checksum thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "279ef31c19ededf577bfd12dfae728040a21f635b06a24cd670ff510edd38963" +"checksum time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "d825be0eb33fda1a7e68012d51e9c7f451dc1a69391e7fdc197060bb8c56667b" +"checksum tinycdb 0.0.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1c37d5f9e60cfdf512708a4461ad1514b8c70c2647a560a67381d55ba191512d" +"checksum tinycdb-sys 0.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "699ede6e96b7a3ce84746111e875b7f6ecbd9dd23d30300594a7e7aa69f57299" +"checksum ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd2be2d6639d0f8fe6cdda291ad456e23629558d466e2789d2c3e9892bda285d" +"checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" +"checksum unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" +"checksum utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "662fab6525a98beff2921d7f61a39e7d59e0b425ebc7d0d9e66d316e55124122" +"checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" +"checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +"checksum wait-timeout 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "b9f3bf741a801531993db6478b95682117471f76916f5e690dd8d45395b09349" +"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" +"checksum winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "773ef9dcc5f24b7d850d0ff101e542ff24c3b090a9768e03ff889fdef41f00fd" +"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +"checksum wincolor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "eeb06499a3a4d44302791052df005d5232b927ed1a9658146d842165c4de7767" diff --git a/src/storage/cdb/Cargo.toml b/src/storage/cdb/Cargo.toml new file mode 100644 index 000000000..701dfa663 --- /dev/null +++ b/src/storage/cdb/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "cdb_rs" +version = "0.1.0" +authors = ["Jonathan D. Simms "] +publish = false + +[lib] +name = "cdb_rs" +crate-type = ["rlib","dylib"] + +[dependencies] +bytes = "~0.4" +rand = "~0.5" +memmap = "~0.6.2" +itertools = "~0.6" +log = "~0.4" +env_logger = "~0.5.10" +tempfile = "~3" +proptest = "~0.7.1" +clap = "~2.31.0" +rust-crypto = "~0.2.36" +libc = "~0.2.42" +tinycdb = "~0.0.7" + +# [dependencies.tinycdb] +# git = "https://github.com/andrew-d/tinycdb-rs" + +[profile.release] +opt-level = 3 +debug = true +rpath = false +lto = true +debug-assertions = false +codegen-units = 1 + +[profile.dev] +debug = true +opt-level = 0 diff --git a/src/storage/cdb/Makefile b/src/storage/cdb/Makefile new file mode 100644 index 000000000..365ae3521 --- /dev/null +++ b/src/storage/cdb/Makefile @@ -0,0 +1,14 @@ +build: + cargo build + +dict.cdb: + scripts/mkcdb.sh + +benchmark: dict.cdb + cargo run --release --bin cdb_rs -- dict.cdb + +clean: + rm -f dict.cdb + +.DEFAULT_GOAL := benchmark +.PHONY: build dict.cdb clean diff --git a/src/storage/cdb/scripts/mkcdb.sh b/src/storage/cdb/scripts/mkcdb.sh new file mode 100755 index 000000000..82ce06564 --- /dev/null +++ b/src/storage/cdb/scripts/mkcdb.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -euo pipefail +IFS=$'\n\t' + +die() { echo "fatal: $*" >&2; exit 1; } + +cd "$(git rev-parse --show-toplevel)" || die "could not cd to top level" + +TEMP="$(mktemp -d -t TEMP.XXXXXXX)" || die "failed to make tmpdir" +cleanup() { [[ -n "${TEMP:-}" ]] && rm -rf "${TEMP}"; } +trap cleanup EXIT + + +INPUT="$TEMP/cdbin" + +NUM_RECS=${NUM_RECS:-5000} + +cargo run --release --bin generatecdb |head -n $NUM_RECS|pv > "$INPUT" +echo '' >> $INPUT + +cdb -c dict.cdb "$INPUT" diff --git a/src/storage/cdb/src/bin/generatecdb.rs b/src/storage/cdb/src/bin/generatecdb.rs new file mode 100644 index 000000000..3cfbeabea --- /dev/null +++ b/src/storage/cdb/src/bin/generatecdb.rs @@ -0,0 +1,56 @@ +extern crate cdb_rs; +extern crate rand; +#[macro_use] extern crate clap; +use clap::ArgMatches; +use rand::{thread_rng,Rng}; +use rand::distributions::Alphanumeric; +use std::io; +use std::io::Write; + + +fn alpha(min: usize, max: usize) -> String { + thread_rng().sample_iter(&Alphanumeric).take( + if min == max { min } else { thread_rng().gen_range(min, max) } + ).collect() +} + +const MIN_KEY_SIZE: usize = 8; +const MIN_VAL_SIZE: usize = 8; +const MAX_KEY_SIZE: usize = 256; +const MAX_VAL_SIZE: usize = 1024 * 1024; +const DEFAULT_N_RECORDS: usize = 1000; + +fn main() { + let matches: ArgMatches = clap_app!(randoread => + (version: "0.1.0") + (author: "Jonathan Simms ") + (@arg ENTRIES: -n --nrecords [N] "number of records to generate") + (@arg MINKEY: -k --minkey [N] "min key size") + (@arg MAXKEY: -K --maxkey [N] "max key size") + (@arg MINVAL: -v --minval [N] "min val size") + (@arg MAXVAL: -V --maxval [N] "max val size") + ).get_matches(); + + let argval = |argname,default| { + match matches.value_of(argname) { + Some(str) => str.parse::().unwrap(), + None => default, + } + }; + + let min_k: usize = argval("MINKEY", MIN_KEY_SIZE); + let max_k: usize = argval("MAXKEY", MAX_KEY_SIZE); + let min_v: usize = argval("MINVAL", MIN_VAL_SIZE); + let max_v: usize = argval("MAXVAL", MAX_VAL_SIZE); + let num_rec: usize = argval("ENTRIES", DEFAULT_N_RECORDS); + + let mut i: usize = 0; + while i < num_rec { + let k: String = alpha(min_k, max_k); + let v: String = alpha(min_v, max_v); + + writeln!(io::stdout(), "+{},{}:{}->{}", k.len(), v.len(), k, v).unwrap(); + i += 1; + } + writeln!(io::stdout(), "").unwrap(); +} diff --git a/src/storage/cdb/src/cdb/cdb_c.rs b/src/storage/cdb/src/cdb/cdb_c.rs new file mode 100644 index 000000000..0bfb92e9f --- /dev/null +++ b/src/storage/cdb/src/cdb/cdb_c.rs @@ -0,0 +1,40 @@ + +use std::ffi::CStr; +use std::os::raw::c_char; + +use super::{CDB, Result}; + +#[repr(C)] +pub struct CDBHandle { + inner: Box +} + +#[no_mangle] +pub extern "C" fn cdb_rs_create(path: *const c_char) -> Option<*mut CDBHandle> { + assert!(!path.is_null()); + + let f = || -> Result> { + let cpath = unsafe { CStr::from_ptr(path) }; + + let path = cpath.to_str()?; + let inner = Box::new(CDB::stdio(path)?); + let handle = CDBHandle{inner}; + + Ok(Box::new(handle)) + }; + + match f() { + Ok(bhandle) => Some(Box::into_raw(bhandle)), + Err(err) => { + error!("failed to create CDBHandle: {:?}", err); + None + } + } +} + +#[no_mangle] +pub extern "C" fn cdb_rs_destroy(handle: *mut CDBHandle) { + unsafe { + drop(Box::from_raw(handle)); + } +} diff --git a/src/storage/cdb/src/cdb/errors.rs b/src/storage/cdb/src/cdb/errors.rs new file mode 100644 index 000000000..74f95104a --- /dev/null +++ b/src/storage/cdb/src/cdb/errors.rs @@ -0,0 +1,67 @@ +use std::error; +use std::fmt; +use std::io; +use std::num::ParseIntError; +use std::str::Utf8Error; +use std::ffi::NulError; + +#[derive(Debug)] +pub enum CDBError { + IOError(io::Error), + UTF8Error(::std::str::Utf8Error), + ParseError(ParseIntError), + NulError(NulError), +} + +impl From for CDBError { + fn from(err: ParseIntError) -> CDBError { + CDBError::ParseError(err) + } +} + +impl From for CDBError { + fn from(err: Utf8Error) -> CDBError { + CDBError::UTF8Error(err) + } +} + +impl From for CDBError { + fn from(err: io::Error) -> CDBError { + CDBError::IOError(err) + } +} + +impl From for CDBError { + fn from(err: NulError) -> CDBError { CDBError::NulError(err) } +} + +impl error::Error for CDBError { + fn description(&self) -> &str { + match *self { + CDBError::IOError(ref err) => err.description(), + CDBError::UTF8Error(ref err) => err.description(), + CDBError::ParseError(ref err) => err.description(), + CDBError::NulError(ref err) => err.description(), + } + } + + fn cause(&self) -> Option<&error::Error> { + match *self { + CDBError::IOError(ref err) => Some(err), + CDBError::UTF8Error(ref err) => Some(err), + CDBError::ParseError(ref err) => Some(err), + CDBError::NulError(ref err) => Some(err), + } + } +} + +impl fmt::Display for CDBError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match *self { + CDBError::IOError(ref err) => err.fmt(f), + CDBError::UTF8Error(ref err) => err.fmt(f), + CDBError::ParseError(ref err) => err.fmt(f), + CDBError::NulError(ref err) => err.fmt(f), + } + } +} diff --git a/src/storage/cdb/src/cdb/input.rs b/src/storage/cdb/src/cdb/input.rs new file mode 100644 index 000000000..28970e1a8 --- /dev/null +++ b/src/storage/cdb/src/cdb/input.rs @@ -0,0 +1,156 @@ +use super::KV; +use super::errors::CDBError; +use bytes::*; +use std::io::BufReader; +use std::io::prelude::*; +use std::str; + +struct KVSizes(usize, usize); + +const PLUS: u8 = 0x2b; // ASCII '+' +const COMMA: u8 = 0x2c; // ASCII ',' +const COLON: u8 = 0x3a; // ASCII ':' +const NL: u8 = 0x0a; // ASCII '\n' + +fn parse_digits(buf: &[u8]) -> Result { + str::from_utf8(&buf) + .map_err(|err| CDBError::UTF8Error(err)) + .and_then(|str| { + str.parse::() + .map_err(|err| CDBError::ParseError(err)) + }) +} + +const ARROW_BYTES: &[u8; 2] = b"->"; + +// format: +1,3:a->xyz\n +fn read_begin(input: &mut BufReader) -> Result, CDBError> { + let mut buf = vec![0u8; 1]; + + // consume a '+' + input.read_exact(&mut buf)?; + eprintln!("read_begin: {:?}", buf[0] as char); + + match buf[0] { + PLUS => Ok(Some(())), + NL => Ok(None), + wat => panic!("encountered unexpected char: {:?}", wat as char), + } +} + +fn read_sizes(input: &mut BufReader) -> Result { + let mut buf: Vec = Vec::new(); + + let r = input.read_until(COMMA, &mut buf)?; + + assert!(r > 0); + assert_eq!(COMMA, buf.pop().unwrap()); + + let k = parse_digits(&buf).unwrap(); + buf.clear(); + + let r = input.read_until(COLON, &mut buf)?; + + assert!(r > 0); + assert_eq!(COLON, buf.pop().unwrap()); + let v = parse_digits(&buf)?; + + Ok(KVSizes(k, v)) +} + +fn read_kv(input: &mut BufReader, kvs: &KVSizes) -> Result { + let KVSizes(ksize, vsize) = kvs; + + let mut kbytes = vec![0u8; *ksize]; + input.read_exact(&mut kbytes)?; + + eprintln!("read K: {:?}", String::from_utf8(kbytes.clone()).unwrap()); + + // consume the "->" between k and v + let mut arrowbytes: [u8; 2] = [0; 2]; + input.read_exact(&mut arrowbytes)?; + assert_eq!(arrowbytes, *ARROW_BYTES); + + let mut vbytes = vec![0u8; *vsize]; + input.read_exact(&mut vbytes)?; + + eprintln!("read V: {:?}", String::from_utf8(vbytes.clone()).unwrap()); + + let mut newline = vec![0u8; 1]; + input.read_exact(&mut newline)?; + + assert_eq!(newline.len(), 1); + assert_eq!(newline[0], NL); + + Ok(KV { + k: Bytes::from(kbytes), + v: Bytes::from(vbytes), + }) +} + +fn read_one_record(input: &mut BufReader) -> Result, CDBError> { + match read_begin(input)? { + None => Ok(None), + Some(_) => read_sizes(input) + .and_then(|sizes| read_kv(input, &sizes)) + .map(|kv| Some(kv)), + } +} + +pub struct IterParser { + buf: BufReader, +} + +impl Iterator for IterParser { + type Item = Result; + + fn next(&mut self) -> Option<::Item> { + match read_one_record(&mut self.buf) { + Ok(Some(kv)) => Some(Ok(kv)), + Ok(None) => None, + Err(err) => Some(Err(err)), + } + } +} + +// expects input in CDB format '+ks,vs:k->v\n' +pub fn parse(rdr: T) -> IterParser { + IterParser { + buf: BufReader::new(rdr), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parser_iter() { + let reader = Bytes::from("+3,4:cat->ball\n\n").into_buf().reader(); + let recs: Vec> = parse(reader).collect(); + + assert_eq!(recs.len(), 1); + match recs[0] { + Ok(KV { ref k, ref v }) => { + assert_eq!(k, "cat"); + assert_eq!(v, "ball"); + } + Err(ref x) => panic!("should not have errored: {:?}", x), + }; + } + + #[test] + fn parser_read_one_record() { + let reader = Bytes::from("+3,4:cat->ball\n\n").into_buf().reader(); + let one = read_one_record(&mut BufReader::new(reader)); + + match one { + Ok(Some(KV { ref k, ref v })) => { + assert_eq!(k, "cat"); + assert_eq!(v, "ball"); + } + Ok(None) => panic!("got None expected Some"), + Err(ref x) => panic!("should not have errored: {:?}", x), + } + } +} diff --git a/src/storage/cdb/src/cdb/mod.rs b/src/storage/cdb/src/cdb/mod.rs new file mode 100644 index 000000000..16f63fc18 --- /dev/null +++ b/src/storage/cdb/src/cdb/mod.rs @@ -0,0 +1,436 @@ +use bytes::{Buf, Bytes, IntoBuf}; +use std::fmt; +use std::io; +use std::io::Write; +use std::result; + +pub mod randoread; +pub mod errors; +pub mod input; +pub mod writer; +pub mod storage; + +pub mod cdb_c; + +use self::storage::SliceFactory; + +pub use self::errors::CDBError; + +pub const STARTING_HASH: u32 = 5381; +const MAIN_TABLE_SIZE: usize = 256; +const MAIN_TABLE_SIZE_BYTES: usize = 2048; +const END_TABLE_ENTRY_SIZE: usize = 8; +const DATA_HEADER_SIZE: usize = 8; + +pub type Result = result::Result; + +// idea from https://raw.githubusercontent.com/jothan/cordoba/master/src/lib.rs +#[derive(Copy, Clone, Eq, PartialEq)] +struct CDBHash(u32); + +impl CDBHash { + fn new(bytes: &[u8]) -> Self { + let mut h = STARTING_HASH; + + for b in bytes { + // wrapping here is explicitly for allowing overflow semantics: + // + // Operations like + on u32 values is intended to never overflow, + // and in some debug configurations overflow is detected and results in a panic. + // While most arithmetic falls into this category, some code explicitly expects + // and relies upon modular arithmetic (e.g., hashing) + // + h = h.wrapping_shl(5).wrapping_add(h) ^ (*b as u32) + } + CDBHash(h) + } + + fn table(&self) -> usize { + self.0 as usize % MAIN_TABLE_SIZE + } + + fn slot(&self, num_ents: usize) -> usize { + (self.0 as usize >> 8) % num_ents + } +} + +impl fmt::Debug for CDBHash { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "CDBHash(0x{:08x})", self.0) + } +} + +impl<'a> From<&'a CDBHash> for usize { + fn from(h: &'a CDBHash) -> Self { + h.0 as usize + } +} + +impl<'a> From<&'a CDBHash> for u32 { + fn from(h: &'a CDBHash) -> Self { + h.0 + } +} + +#[derive(Copy, Clone)] +struct Bucket { + ptr: usize, + num_ents: usize, +} + +impl fmt::Debug for Bucket { + fn fmt(&self, f: &mut fmt::Formatter) -> result::Result<(), fmt::Error> { + write!( + f, + "TableRec {{ ptr: {:>#010x}, num_ents: {:>#010x} }}", + self.ptr, self.num_ents + ) + } +} + +impl Bucket { + // returns the offset into the db of entry n of this bucket. + // panics if n >= num_ents + fn entry_n_pos<'a>(&'a self, n: usize) -> IndexEntryPos { + assert!(n < self.num_ents); + IndexEntryPos(self.ptr + (n * END_TABLE_ENTRY_SIZE)) + } +} + +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +struct IndexEntryPos(usize); + +impl From for usize { + fn from(n: IndexEntryPos) -> Self { + n.0 + } +} + +#[derive(Clone, Debug)] +pub struct KV { + k: Bytes, + v: Bytes, +} + +impl KV { + #[allow(dead_code)] + fn dump(&self, w: &mut impl io::Write) -> io::Result<()> { + write!(w, "+{},{}:", self.k.len(), self.v.len())?; + w.write(self.k.as_ref())?; + write!(w, "->")?; + w.write(self.v.as_ref())?; + write!(w, "\n") + } +} + + +struct IndexEntry { + hash: CDBHash, // the hash of the stored key + ptr: usize, // pointer to the absolute position of the data in the db +} + +pub struct KVIter { + cdb: Box, + bkt_idx: usize, + entry_n: usize, + bkt: Bucket, +} + +impl KVIter { + fn new(cdb: Box) -> Result { + let bkt = cdb.bucket_at(0)?; + Ok(KVIter{cdb, bkt_idx: 0, entry_n: 0, bkt}) + } +} + +impl Iterator for KVIter { + type Item = Result; + + fn next(&mut self) -> Option { + loop { + if self.bkt_idx >= MAIN_TABLE_SIZE { + return None + } + + if self.entry_n >= self.bkt.num_ents { + self.bkt_idx += 1; + self.entry_n = 0; + if self.bkt_idx < MAIN_TABLE_SIZE { + match self.cdb.bucket_at(self.bkt_idx) { + Ok(bkt) => self.bkt = bkt, + Err(err) => return Some(Err(err)), + } + } + continue + } + + let idx_ent = + match self.cdb.index_entry_at(self.bkt.entry_n_pos(self.entry_n)) { + Ok(index_entry) => index_entry, + Err(err) => return Some(Err(err)), + }; + + self.entry_n += 1; + + if idx_ent.ptr == 0 { + continue + } else { + return match self.cdb.get_kv(idx_ent) { + Ok(kv) => Some(Ok(kv)), + Err(err) => Some(Err(err)), + } + } + } + } +} + +pub struct CDB { + data: SliceFactory, +} + +impl Clone for CDB { + fn clone(&self) -> Self { + CDB{data: self.data.clone()} + } +} + +impl CDB { + pub fn new(sf: SliceFactory) -> CDB { CDB{data: sf} } + + pub fn stdio(path: &str) -> Result { + Ok(CDB::new(SliceFactory::make_filewrap(path)?)) + } + + pub fn mmap(path: &str) -> Result { + Ok(CDB::new(SliceFactory::make_map(path)?)) + } + + pub fn load(path: &str) -> Result { + Ok(CDB::new(SliceFactory::load(path)?)) + } + + pub fn kvs_iter(&self) -> Result { + Ok(KVIter::new(Box::new(self.clone()))?) + } + + #[inline] + fn bucket_at(&self, idx: usize) -> Result { + assert!(idx < MAIN_TABLE_SIZE); + + let off = 8 * idx; + + let b = self.data.slice(off, off + 8)?; + assert_eq!(b.len(), 8); + trace!("bucket_at idx: {}, got buf: {:?}", idx, b); + + let mut buf = b.into_buf(); + + let ptr = buf.get_u32_le() as usize; + let num_ents = buf.get_u32_le() as usize; + + Ok(Bucket{ptr, num_ents}) + } + + + + // returns the index entry at absolute position 'pos' in the db + #[inline] + fn index_entry_at(&self, pos: IndexEntryPos) -> Result { + let pos: usize = pos.into(); + + if pos < MAIN_TABLE_SIZE_BYTES { + panic!("position {:?} was in the main table!", pos) + } + + let mut b = self.data.slice(pos, pos + 8)?.into_buf(); + let hash = CDBHash(b.get_u32_le()); + let ptr = b.get_u32_le() as usize; + + Ok(IndexEntry{hash, ptr}) + } + + #[inline] + fn get_kv(&self, ie: IndexEntry) -> Result { + let b = self.data.slice(ie.ptr, ie.ptr + DATA_HEADER_SIZE)?; + + let ksize = b.slice_to(4).into_buf().get_u32_le() as usize; + let vsize = b.slice_from(4).into_buf().get_u32_le() as usize; + + let kstart = ie.ptr + DATA_HEADER_SIZE; + let vstart = kstart + ksize; + + let k = self.data.slice(kstart, kstart + ksize)?; + let v = self.data.slice(vstart, vstart + vsize)?; + + Ok(KV{k: Bytes::from(k), v: Bytes::from(v)}) + } + + pub fn get(&self, key: &[u8], buf: &mut Vec) -> Result> { + let key = key.into(); + let hash = CDBHash::new(key); + let bucket = self.bucket_at(hash.table())?; + + if bucket.num_ents == 0 { + trace!("bucket empty, returning none"); + return Ok(None); + } + + let slot = hash.slot(bucket.num_ents); + + for x in 0..bucket.num_ents { + let index_entry_pos = bucket.entry_n_pos((x + slot) % bucket.num_ents); + + let idx_ent = self.index_entry_at(index_entry_pos)?; + + if idx_ent.ptr == 0 { + return Ok(None); + } else if idx_ent.hash == hash { + let kv = self.get_kv(idx_ent)?; + if &kv.k[..] == key { + buf.write_all(&kv.k[..]).unwrap(); + return Ok(Some(kv.k.len())); + } else { + continue; + } + } + } + + Ok(None) + } +} + +#[cfg(test)] +mod tests { + use env_logger; + use proptest::collection::vec; + use proptest::prelude::*; + use proptest::string; + use std::collections::hash_set; + use std::fs::File; + use std::fs::remove_file; + use std::io::{BufRead, BufReader}; + use std::path::Path; + use std::path::PathBuf; + use super::*; + use tempfile::NamedTempFile; + use tinycdb::Cdb as TCDB; + + fn arb_string_slice<'a>() -> BoxedStrategy> { + let st = string::string_regex("[a-z]+").unwrap(); + vec(st, 10..1000).boxed() + } + + struct QueryResult(String, Option); + + #[allow(dead_code)] + fn create_temp_cdb<'a>(kvs: &Vec<(String, String)>) -> Result { + let path: PathBuf; + + { + let ntf = NamedTempFile::new()?; + remove_file(ntf.path())?; + path = ntf.path().to_owned(); + } + + let mut dupcheck = hash_set::HashSet::new(); + + TCDB::new(path.as_ref(), |c| { + let ys = kvs.to_owned(); + for (k, v) in ys { + let kk = k.clone(); + let vv = v.clone(); + + if !dupcheck.contains(&k) { + dupcheck.insert(k); + c.add(kk.as_ref(), vv.as_ref()).unwrap(); + } + } + }).unwrap(); + + let sf = self::storage::SliceFactory::load(path.to_str().unwrap())?; + + Ok(sf) + } + + proptest! { + #[test] + fn qc_key_and_value_retrieval(ref xs in arb_string_slice()) { + let cdb = CDB::new(make_temp_cdb_single_vals(&xs)); + + for QueryResult(q, r) in read_keys(&cdb, &xs) { + prop_assert_eq!( + Some(q), + r + ); + } + } + } + + type QueryResultIter<'a> = Box + 'a>; + + fn read_keys<'a>(cdb: &'a CDB, xs: &'a Vec) -> QueryResultIter<'a> { + Box::new(xs.iter().map(move |x| { + let mut buf = Vec::with_capacity(1024 * 1024); + let res = cdb.get(x.as_ref(), &mut buf).unwrap(); + QueryResult( + x.clone(), + res.map(|_| String::from_utf8(buf).unwrap()), + ) + })) + } + + #[allow(dead_code)] + fn make_temp_cdb_single_vals(xs: &Vec) -> SliceFactory { + let kvs: Vec<(String, String)> = + xs.iter().map(|k| (k.to_owned(), k.to_owned())).collect(); + create_temp_cdb(&kvs).unwrap() + } + + #[test] + fn read_small_list() { + env_logger::try_init().unwrap(); + + let strings = vec![ + "shngcmfkqjtvhnbgfcvbm", + "qjflpsvacyhsgxykbvarbvmxapufmdt", + "a", + "a", + "a", + "a", + "a", + "a", + "xfjhaqjkcjiepmcbhopgpxwwth", + "a", + "a", + ]; + let arg = strings.iter().map(|s| (*s).to_owned()).collect(); + + let sf = make_temp_cdb_single_vals(&arg); + let cdb = CDB::new(sf); + + for QueryResult(q, r) in read_keys(&cdb, &arg) { + assert_eq!(Some(q), r); + } + } + + #[test] + fn test_with_dictionary() { + let mut args: Vec = Vec::new(); + + { + let f = File::open(Path::new("/usr/share/dict/words")).unwrap(); + let bufr = BufReader::new(&f); + + for line in bufr.lines() { + let word = line.unwrap(); + args.push(word.to_owned()); + } + } + + let cdb = CDB::new(make_temp_cdb_single_vals(&args)); + + for QueryResult(q, r) in read_keys(&cdb, &args) { + assert_eq!(Some(q), r); + } + } +} diff --git a/src/storage/cdb/src/cdb/randoread.rs b/src/storage/cdb/src/cdb/randoread.rs new file mode 100644 index 000000000..ab026eb80 --- /dev/null +++ b/src/storage/cdb/src/cdb/randoread.rs @@ -0,0 +1,113 @@ +use bytes::Bytes; +use rand::{thread_rng, Rng}; +use std::time::{Duration, Instant}; + +use super::Result; + +#[derive(Clone, Copy, Debug)] +pub struct RandoConfig { + // a number from [0.0, 1.0) that controls the likelihood that any + // particular key will be chosen to test with. 0.3 by default + pub probability: f32, + + // max number of keys to test with, defaults to 10k + pub max_keys: usize, + + // number of iterations to do: default 10k + pub iters: u64, + + pub use_mmap: bool, + + pub use_stdio: bool, +} + +impl RandoConfig { + pub fn new() -> RandoConfig { + RandoConfig { + probability: 0.3, + max_keys: 10_000, + iters: 10_000, + use_mmap: false, + use_stdio: false, + } + } + + pub fn probability<'a>(&'a mut self, prob: f32) -> &'a mut RandoConfig { + self.probability = prob; + self + } + + pub fn max_keys<'a>(&'a mut self, max: usize) -> &'a mut RandoConfig { + self.max_keys = max; + self + } + + pub fn iters<'a>(&'a mut self, num_iter: u64) -> &'a mut RandoConfig { + self.iters = num_iter; + self + } + + pub fn use_mmap<'a>(&'a mut self, b: bool) -> &'a mut RandoConfig { + self.use_mmap = b; + self + } + + pub fn use_stdio<'a>(&'a mut self, b: bool) -> &'a mut RandoConfig { + self.use_stdio = b; + self + } +} + + +pub fn run(db: &super::CDB, config: &RandoConfig) -> Result { + let mut rng = thread_rng(); + + let mut keys = { + let mut ks: Vec = + db.kvs_iter()? + .map(|kv| kv.unwrap().k) + .collect(); + + ks.shrink_to_fit(); + ks + }; + + rng.shuffle(&mut keys); + + let keyiter = + keys.iter() + .take(config.iters as usize) + .cycle() + .take(config.iters as usize); + + eprintln!("starting test using {} sampled keys", config.iters); + let start = Instant::now(); + + let mut hit = 0; + let mut miss = 0; + let mut bytes = 0; + + let mut buf: Vec = Vec::with_capacity(1024 * 1024); + + for k in keyiter { + buf.clear(); + if db.get(&k[..], &mut buf)?.is_some() { + hit += 1; + bytes += buf.len(); + } else { + miss += 1 + } + } + + let hitrate = (hit as f64 / config.iters as f64) * 100.0; + + debug!( + "hit: {}, miss: {}, ratio: {:.3}%, bytes: {}", + hit, + miss, + hitrate, + bytes + ); + + Ok(start.elapsed()) +} diff --git a/src/storage/cdb/src/cdb/storage.rs b/src/storage/cdb/src/cdb/storage.rs new file mode 100644 index 000000000..51afb3dba --- /dev/null +++ b/src/storage/cdb/src/cdb/storage.rs @@ -0,0 +1,235 @@ +use bytes::{Buf, Bytes, BytesMut}; +use crypto::digest::Digest; +use crypto::md5::Md5; +use memmap::{Mmap, MmapOptions}; +use std::cell::RefCell; +use std::fs::File; +use std::io::{Cursor, Read}; +use std::ops::Deref; +use std::os::unix::fs::FileExt; +use std::sync::Arc; +use super::Result; + +pub enum SliceFactory { + HeapStorage(Bytes), + MmapStorage(MMapWrap), + StdioStorage(FileWrap), +} + +const BUF_LEN: usize = 8192; + +pub fn readybuf(size: usize) -> BytesMut { + let mut b = BytesMut::with_capacity(size); + b.resize(size, 0); + b +} + +impl SliceFactory { + pub fn load(path: &str) -> Result { + let mut f = File::open(path)?; + let mut buffer = Vec::new(); + f.read_to_end(&mut buffer)?; + Ok(SliceFactory::HeapStorage(Bytes::from(buffer))) + } + + pub fn make_map(path: &str) -> Result { + let f = File::open(path)?; + let mmap: Mmap = unsafe { MmapOptions::new().map(&f)? }; + + let mut buf = [0u8; BUF_LEN]; + let mut count = 0; + let mut md5 = Md5::new(); + + debug!("begin pretouch pages"); + { + let mut cur = Cursor::new(&mmap[..]); + loop { + let remain = cur.remaining(); + if remain < BUF_LEN { + let mut buf = readybuf(remain); + cur.copy_to_slice(&mut buf[..]); + count += buf.len(); + md5.input(&buf); + break; + } else { + cur.copy_to_slice(&mut buf); + count += BUF_LEN; + md5.input(&buf); + } + } + } + debug!( + "end pretouch pages: {} bytes, md5: {}", + count, + md5.result_str() + ); + + Ok(SliceFactory::MmapStorage(MMapWrap::new(mmap))) + } + + pub fn make_filewrap(path: &str) -> Result { + Ok(SliceFactory::StdioStorage(FileWrap::open(path)?)) + } + + pub fn slice(&self, start: usize, end: usize) -> Result { + assert!(end >= start); + + if end == start { + return Ok(Bytes::new()); + } + + let range_len = end - start; + + match self { + SliceFactory::HeapStorage(bytes) => Ok(Bytes::from(&bytes[start..end])), + SliceFactory::MmapStorage(mmap) => { + let mut v = Vec::with_capacity(range_len); + v.extend_from_slice(&mmap[start..end]); + Ok(Bytes::from(v)) + } + SliceFactory::StdioStorage(filewrap) => filewrap.slice(start, end), + } + } +} + +impl Clone for SliceFactory { + fn clone(&self) -> Self { + match self { + SliceFactory::HeapStorage(bytes) => SliceFactory::HeapStorage(bytes.clone()), + SliceFactory::MmapStorage(mmap) => SliceFactory::MmapStorage(mmap.clone()), + SliceFactory::StdioStorage(fw) => SliceFactory::StdioStorage(fw.clone()), + } + } +} + +pub struct MMapWrap { + inner: Arc +} + +impl MMapWrap { + fn new(m: Mmap) -> MMapWrap { + MMapWrap{inner: Arc::new(m)} + } +} + +impl Deref for MMapWrap { + type Target = Mmap; + + fn deref(&self) -> &::Target { + self.inner.as_ref() + } +} + +impl Clone for MMapWrap { + fn clone(&self) -> Self { + MMapWrap{inner: self.inner.clone()} + } +} + +pub struct FileWrap { + inner: RefCell, + path: String, +} + +impl FileWrap { + fn new(f: File, path: &str) -> Self { + FileWrap { + inner: RefCell::new(f), + path: path.to_string(), + } + } + + fn open(path: &str) -> Result { + Ok(FileWrap::new(File::open(path)?, path)) + } + + fn slice(&self, start: usize, end: usize) -> Result { + assert!(end >= start); + let mut buf = readybuf(end - start); + { + let fp = self.inner.borrow_mut(); + fp.read_at(&mut buf, start as u64)?; + trace!("read: {:?}", buf); + } + Ok(Bytes::from(buf)) + } + + #[cfg(test)] + fn temp() -> Result { + use tempfile::*; + + let tmp = NamedTempFile::new()?; + let fw = FileWrap::new(tmp.as_file().try_clone()?, tmp.path().to_str().unwrap()); + Ok(fw) + } +} + +impl Clone for FileWrap { + fn clone(&self) -> Self { + FileWrap::open(self.path.as_ref()).unwrap() + } +} + +struct BMString(BytesMut); + +impl ToString for BMString { + fn to_string(&self) -> String { + String::from(self) + } +} + +impl<'a> From<&'a BMString> for String { + fn from(bm: &'a BMString) -> Self { + String::from_utf8(bm.0.to_vec()).unwrap() + } +} + +#[cfg(test)] +mod tests { + use std::fs::File; + use std::io::prelude::*; + use super::*; + use tempfile; + + fn assert_ok(f: T) + where + T: Fn() -> Result<()>, + { + f().unwrap() + } + + #[test] + fn basic_file_io_sanity() { + assert_ok(|| { + let mut tmp: File = tempfile::tempfile()?; + + tmp.write_all("abcdefghijklmnopqrstuvwxyz".as_bytes())?; + tmp.sync_all()?; + + let mut buf = BytesMut::with_capacity(3); + buf.resize(3, 0); + let n = tmp.read_at(&mut buf, 23)?; + assert_eq!(n, 3); + assert_eq!(&buf[..], "xyz".as_bytes()); + Ok(()) + }) + } + + #[test] + fn file_wrap_slice_test() { + assert_ok(||{ + let fw = FileWrap::temp()?; + + { + let mut f = fw.inner.borrow_mut(); + f.write_all("abcdefghijklmnopqrstuvwxyz".as_bytes())?; + f.sync_all()?; + } + + assert_eq!(fw.slice(3, 5)?, "de".as_bytes()); + Ok(()) + }) + } +} + + diff --git a/src/storage/cdb/src/cdb/writer.rs b/src/storage/cdb/src/cdb/writer.rs new file mode 100644 index 000000000..cac3dff1f --- /dev/null +++ b/src/storage/cdb/src/cdb/writer.rs @@ -0,0 +1,3 @@ +#[allow(dead_code)] + +pub struct Writer {} diff --git a/src/storage/cdb/src/lib.rs b/src/storage/cdb/src/lib.rs new file mode 100644 index 000000000..890f16cdc --- /dev/null +++ b/src/storage/cdb/src/lib.rs @@ -0,0 +1,19 @@ +extern crate bytes; +extern crate rand; +extern crate memmap; +extern crate itertools; + +#[macro_use] extern crate log; +extern crate env_logger; +extern crate tempfile; + +#[macro_use] extern crate proptest; + +extern crate tinycdb; +extern crate clap; +extern crate crypto; +extern crate libc; + +pub mod cdb; + +pub use cdb::cdb_c::{cdb_rs_create, cdb_rs_destroy, CDBHandle}; diff --git a/src/storage/cdb/src/main.rs b/src/storage/cdb/src/main.rs new file mode 100644 index 000000000..c3fd197f8 --- /dev/null +++ b/src/storage/cdb/src/main.rs @@ -0,0 +1,102 @@ +extern crate cdb_rs; +#[macro_use] +extern crate log; +extern crate env_logger; +#[macro_use] extern crate clap; +extern crate memmap; + +use std::time::Duration; + +use cdb_rs::cdb; +use cdb_rs::cdb::storage::SliceFactory; +use cdb_rs::cdb::randoread::RandoConfig; + +use cdb_rs::cdb::Result; + +use clap::ArgMatches; + +fn dur2sec(d: &Duration) -> f64 { + d.as_secs() as f64 + (d.subsec_nanos() as f64 * 1e-9) +} + +fn randoread(filename: &str, config: &RandoConfig) -> Result<()> { + let sf: SliceFactory; + + let db = + if config.use_mmap { + cdb::CDB::new(SliceFactory::make_map(filename)?) + } else { + { + if config.use_stdio { + sf = SliceFactory::make_filewrap(filename)?; + } else { + sf = SliceFactory::load(filename)?; + } + } + cdb::CDB::new(sf) + }; + + let d = cdb::randoread::run(&db, &config)?; + let d2f = dur2sec(&d); + let rate = config.iters as f64 / d2f; + + info!("{} iters in {:.3} sec, {:.3} op/sec", config.iters, d2f, rate); + Ok(()) +} + +fn main() { + match env_logger::try_init() { + Ok(_) => "", // yay great + Err(_) => "", // wtfever + }; + + let matches: ArgMatches = clap_app!(randoread => + (version: "0.1.0") + (author: "Jonathan Simms ") + (@arg ITERS: -i --iters [N] "number of iterations to do") + (@arg PROB: -p --probability [N] "probability filter for keys, float [0.0, 1.0)") + (@arg NKEYS: -k --numkeys [N] "max number of keys to test with") + (@arg MMAP: -M --mmap conflicts_with[STDIO] "use alternate mmap implmeentation (experimental on linux)") + (@arg STDIO: -S --stdio conflicts_with[MMAP] "use stdio implementation") + (@arg INPUT: +required "the .cdb file to test") + ).get_matches(); + + let mut rc = RandoConfig::new(); + + if let Some(val) = matches.value_of("ITERS") { + rc.iters(val.parse().unwrap()); + } + + if let Some(p) = matches.value_of("PROB") { + rc.probability(p.parse().unwrap()); + } + + if let Some(p) = matches.value_of("NKEYS") { + rc.max_keys(p.parse().unwrap()); + } + + match matches.occurrences_of("MMAP") { + 0 => rc.use_mmap(false), + _ => rc.use_mmap(true), + }; + + match matches.occurrences_of("STDIO") { + 0 => rc.use_stdio(false), + _ => rc.use_stdio(true), + }; + + + let filename = matches.value_of("INPUT").unwrap(); + + debug!("using config: {:?}", rc); + + std::process::exit( + match randoread(filename, &rc) { + Ok(_) => 0, + Err(err) => { + eprintln!("error: {:?}", err); + 1 + } + } + ); +} diff --git a/test/data_structure/bitmap/CMakeLists.txt b/test/data_structure/bitmap/CMakeLists.txt index c3a722953..0763d73ed 100644 --- a/test/data_structure/bitmap/CMakeLists.txt +++ b/test/data_structure/bitmap/CMakeLists.txt @@ -6,6 +6,7 @@ set(source check_${suite}.c) add_executable(${test_name} ${source}) target_link_libraries(${test_name} ds_${suite}) target_link_libraries(${test_name} ccommon-static ${CHECK_LIBRARIES}) +target_link_libraries(${test_name} pthread subunit m) add_dependencies(check ${test_name}) add_test(${test_name} ${test_name}) diff --git a/test/data_structure/ziplist/CMakeLists.txt b/test/data_structure/ziplist/CMakeLists.txt index 9e91605d2..9d3afd902 100644 --- a/test/data_structure/ziplist/CMakeLists.txt +++ b/test/data_structure/ziplist/CMakeLists.txt @@ -5,7 +5,7 @@ set(source check_${suite}.c) add_executable(${test_name} ${source}) target_link_libraries(${test_name} ds_${suite}) -target_link_libraries(${test_name} ccommon-static ${CHECK_LIBRARIES}) +target_link_libraries(${test_name} ccommon-static ${CHECK_LIBRARIES} pthread subunit m) add_dependencies(check ${test_name}) add_test(${test_name} ${test_name}) diff --git a/test/hotkey/kc_map/CMakeLists.txt b/test/hotkey/kc_map/CMakeLists.txt index 505070af4..7d9cc5ab8 100644 --- a/test/hotkey/kc_map/CMakeLists.txt +++ b/test/hotkey/kc_map/CMakeLists.txt @@ -5,7 +5,7 @@ set(source check_${suite}.c) add_executable(${test_name} ${source}) target_link_libraries(${test_name} hotkey) -target_link_libraries(${test_name} ccommon-static ${CHECK_LIBRARIES}) +target_link_libraries(${test_name} ccommon-static ${CHECK_LIBRARIES} pthread subunit m) add_dependencies(check ${test_name}) add_test(${test_name} ${test_name}) diff --git a/test/hotkey/key_window/CMakeLists.txt b/test/hotkey/key_window/CMakeLists.txt index cd4b688e4..271dadcd6 100644 --- a/test/hotkey/key_window/CMakeLists.txt +++ b/test/hotkey/key_window/CMakeLists.txt @@ -5,7 +5,7 @@ set(source check_${suite}.c) add_executable(${test_name} ${source}) target_link_libraries(${test_name} hotkey) -target_link_libraries(${test_name} ccommon-static ${CHECK_LIBRARIES}) +target_link_libraries(${test_name} ccommon-static ${CHECK_LIBRARIES} pthread subunit m) add_dependencies(check ${test_name}) add_test(${test_name} ${test_name}) diff --git a/test/protocol/admin/CMakeLists.txt b/test/protocol/admin/CMakeLists.txt index beb855e9d..51c016368 100644 --- a/test/protocol/admin/CMakeLists.txt +++ b/test/protocol/admin/CMakeLists.txt @@ -5,7 +5,7 @@ set(source check_${suite}.c) add_executable(${test_name} ${source}) target_link_libraries(${test_name} protocol_${suite}) -target_link_libraries(${test_name} ccommon-static ${CHECK_LIBRARIES}) +target_link_libraries(${test_name} ccommon-static ${CHECK_LIBRARIES} pthread subunit m) add_dependencies(check ${test_name}) add_test(${test_name} ${test_name}) diff --git a/test/protocol/data/memcache/CMakeLists.txt b/test/protocol/data/memcache/CMakeLists.txt index 34e4a0293..8d529cd65 100644 --- a/test/protocol/data/memcache/CMakeLists.txt +++ b/test/protocol/data/memcache/CMakeLists.txt @@ -7,7 +7,7 @@ set(source check_${suite}.c) add_executable(${test_name} ${source}) target_link_libraries(${test_name} protocol_${suite}) -target_link_libraries(${test_name} ccommon-static ${CHECK_LIBRARIES}) +target_link_libraries(${test_name} ccommon-static ${CHECK_LIBRARIES} pthread subunit m) add_dependencies(check ${test_name}) add_test(${test_name} ${test_name}) diff --git a/test/protocol/data/redis/CMakeLists.txt b/test/protocol/data/redis/CMakeLists.txt index dff05da00..6d6c0ad85 100644 --- a/test/protocol/data/redis/CMakeLists.txt +++ b/test/protocol/data/redis/CMakeLists.txt @@ -5,7 +5,7 @@ set(source check_${suite}.c) add_executable(${test_name} ${source}) target_link_libraries(${test_name} protocol_${suite}) -target_link_libraries(${test_name} ccommon-static ${CHECK_LIBRARIES}) +target_link_libraries(${test_name} ccommon-static ${CHECK_LIBRARIES} pthread subunit m) add_dependencies(check ${test_name}) add_test(${test_name} ${test_name}) diff --git a/test/storage/cuckoo/CMakeLists.txt b/test/storage/cuckoo/CMakeLists.txt index 2c4a799b8..1d2684bc7 100644 --- a/test/storage/cuckoo/CMakeLists.txt +++ b/test/storage/cuckoo/CMakeLists.txt @@ -7,6 +7,7 @@ add_executable(${test_name} ${source}) target_link_libraries(${test_name} ${suite}) target_link_libraries(${test_name} ccommon-static ${CHECK_LIBRARIES}) target_link_libraries(${test_name} time) +target_link_libraries(${test_name} pthread subunit m) add_dependencies(check ${test_name}) add_test(${test_name} ${test_name}) diff --git a/test/storage/slab/CMakeLists.txt b/test/storage/slab/CMakeLists.txt index 24936c1f4..c9408e4bd 100644 --- a/test/storage/slab/CMakeLists.txt +++ b/test/storage/slab/CMakeLists.txt @@ -7,6 +7,7 @@ add_executable(${test_name} ${source}) target_link_libraries(${test_name} ${suite}) target_link_libraries(${test_name} ccommon-static ${CHECK_LIBRARIES}) target_link_libraries(${test_name} time) +target_link_libraries(${test_name} pthread subunit m) add_dependencies(check ${test_name}) add_test(${test_name} ${test_name}) From 98a56533c7d7a6a5120fca6d8aa0ff82b8a5bab9 Mon Sep 17 00:00:00 2001 From: "Jonathan D. Simms" Date: Wed, 20 Jun 2018 21:12:56 +0000 Subject: [PATCH 02/24] CDB with an FFI-friendly implementation straight copy-pasta of src/server/{twemcache => cdb} checkpoint - squash remove flag checkpoint create cdb workspace so shared libs link properly gah i need someone who knows cmake failling, about to try rando CMakeRust module got further with this Rust macro stuff omg, actual progress oops, add the macros rearrange and clean up dependencies clean up cruft compiles and links on debian beginning pelikan side of integration checkpoint generate both static and shared targets shared library building ftw? this is fine. checkpoint - string still getting truncated fix build check try and fix build debugging cmake vars checkpoint hi wil revert change to system calling scheme, no diff finally seems to have linked properly checkpoint checkpoint ignore cdb files squash add compile_commands.json to gitignore squash squash remove redundant return hook up cdb_get, munge a response and cleanup missed two places for raw_val need to rework storage strategy shit, revert this mess Revert "shit, revert this mess" This reverts commit e369bd7769281393c38f50119f3b650890ed294a. gahhhhhh ugh, this is sketchy bindgen is compiling and linking compiles again, finally --- .gitignore | 5 + CMakeLists.txt | 58 ++- cmake/CMakeCargo.cmake | 97 ++++ cmake/CMakeDetermineRustCompiler.cmake | 25 + cmake/CMakeRustCompiler.cmake.in | 15 + cmake/CMakeRustInformation.cmake | 106 ++++ cmake/CMakeTestRustCompiler.cmake | 3 + cmake/FindRust.cmake | 75 +++ config/cdb.conf | 17 + deps/ccommon/CMakeLists.txt | 1 - deps/ccommon/src/CMakeLists.txt | 6 +- deps/ccommon/src/cc_debug.c | 8 +- deps/ccommon/test/array/CMakeLists.txt | 2 +- src/server/CMakeLists.txt | 4 + src/server/cdb/CMakeLists.txt | 32 ++ src/server/cdb/admin/CMakeLists.txt | 4 + src/server/cdb/admin/process.c | 113 ++++ src/server/cdb/admin/process.h | 4 + src/server/cdb/data/CMakeLists.txt | 4 + src/server/cdb/data/process.c | 491 ++++++++++++++++++ src/server/cdb/data/process.h | 81 +++ src/server/cdb/main.c | 235 +++++++++ src/server/cdb/setting.c | 21 + src/server/cdb/setting.h | 53 ++ src/server/cdb/stats.c | 23 + src/server/cdb/stats.h | 41 ++ src/storage/cdb/CMakeLists.txt | 21 +- src/storage/cdb/Cargo.lock | 206 +++++--- src/storage/cdb/Cargo.toml | 33 +- src/storage/cdb/cdb.h | 22 + src/storage/cdb/cdb_ffi/CMakeLists.txt | 12 + src/storage/cdb/cdb_ffi/Cargo.toml | 18 + src/storage/cdb/cdb_ffi/build.rs | 33 ++ src/storage/cdb/cdb_ffi/src/lib.rs | 139 +++++ src/storage/cdb/cdb_ffi/wrapper.h | 1 + src/storage/cdb/cdb_rs/Cargo.toml | 22 + .../cdb/{ => cdb_rs}/src/cdb/errors.rs | 0 src/storage/cdb/{ => cdb_rs}/src/cdb/input.rs | 0 src/storage/cdb/{ => cdb_rs}/src/cdb/mod.rs | 143 ++--- .../cdb/{ => cdb_rs}/src/cdb/storage.rs | 61 ++- src/storage/cdb/cdb_rs/src/lib.rs | 14 + src/storage/cdb/src/bin/generatecdb.rs | 56 -- src/storage/cdb/src/cdb/cdb_c.rs | 40 -- src/storage/cdb/src/cdb/randoread.rs | 113 ---- src/storage/cdb/src/cdb/writer.rs | 3 - src/storage/cdb/src/lib.rs | 19 - src/storage/cdb/src/main.rs | 102 ---- 47 files changed, 2013 insertions(+), 569 deletions(-) create mode 100644 cmake/CMakeCargo.cmake create mode 100644 cmake/CMakeDetermineRustCompiler.cmake create mode 100644 cmake/CMakeRustCompiler.cmake.in create mode 100644 cmake/CMakeRustInformation.cmake create mode 100644 cmake/CMakeTestRustCompiler.cmake create mode 100644 cmake/FindRust.cmake create mode 100644 config/cdb.conf create mode 100644 src/server/cdb/CMakeLists.txt create mode 100644 src/server/cdb/admin/CMakeLists.txt create mode 100644 src/server/cdb/admin/process.c create mode 100644 src/server/cdb/admin/process.h create mode 100644 src/server/cdb/data/CMakeLists.txt create mode 100644 src/server/cdb/data/process.c create mode 100644 src/server/cdb/data/process.h create mode 100644 src/server/cdb/main.c create mode 100644 src/server/cdb/setting.c create mode 100644 src/server/cdb/setting.h create mode 100644 src/server/cdb/stats.c create mode 100644 src/server/cdb/stats.h create mode 100644 src/storage/cdb/cdb.h create mode 100644 src/storage/cdb/cdb_ffi/CMakeLists.txt create mode 100644 src/storage/cdb/cdb_ffi/Cargo.toml create mode 100644 src/storage/cdb/cdb_ffi/build.rs create mode 100644 src/storage/cdb/cdb_ffi/src/lib.rs create mode 100644 src/storage/cdb/cdb_ffi/wrapper.h create mode 100644 src/storage/cdb/cdb_rs/Cargo.toml rename src/storage/cdb/{ => cdb_rs}/src/cdb/errors.rs (100%) rename src/storage/cdb/{ => cdb_rs}/src/cdb/input.rs (100%) rename src/storage/cdb/{ => cdb_rs}/src/cdb/mod.rs (72%) rename src/storage/cdb/{ => cdb_rs}/src/cdb/storage.rs (84%) create mode 100644 src/storage/cdb/cdb_rs/src/lib.rs delete mode 100644 src/storage/cdb/src/bin/generatecdb.rs delete mode 100644 src/storage/cdb/src/cdb/cdb_c.rs delete mode 100644 src/storage/cdb/src/cdb/randoread.rs delete mode 100644 src/storage/cdb/src/cdb/writer.rs delete mode 100644 src/storage/cdb/src/lib.rs delete mode 100644 src/storage/cdb/src/main.rs diff --git a/.gitignore b/.gitignore index 0f32ffdb6..f366f88b3 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,8 @@ lcov *.pyc cdb.rs +*.cdb +compile_commands.json + +*.cmd +core diff --git a/CMakeLists.txt b/CMakeLists.txt index 3f2bdb83b..af6cf7539 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,9 +39,11 @@ option(TARGET_REDIS "build redis binary" ON) option(TARGET_SLIMREDIS "build slimredis binary" ON) option(TARGET_SLIMCACHE "build slimcache binary" ON) option(TARGET_TWEMCACHE "build twemcache binary" ON) - +option(TARGET_CDB "build cdb binary" ON) option(TARGET_RESPCLI "build resp-cli binary" ON) +option(RUST_USE_MUSL "build rust deps against musl" OFF) + option(COVERAGE "code coverage" OFF) # Note: duplicate custom targets only works with Makefile generators, will break XCode & VS @@ -50,6 +52,44 @@ set_property(GLOBAL PROPERTY ALLOW_DUPLICATE_CUSTOM_TARGETS 1) set(CMAKE_SKIP_INSTALL_ALL_DEPENDENCY true) set(CMAKE_MACOSX_RPATH 1) +# "Always full RPATH" +# https://gitlab.kitware.com/cmake/community/wikis/doc/cmake/RPATH-handling +# +if(USE_FULL_RPATH) + # use, i.e. don't skip the full RPATH for the build tree + set(CMAKE_SKIP_BUILD_RPATH FALSE) + + # when building, don't use the install RPATH already + # (but later on when installing) + set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) + + set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") + + # add the automatically determined parts of the RPATH + # which point to directories outside the build tree to the install RPATH + set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) + + # the RPATH to be used when installing, but only if it's not a system directory + list(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES "${CMAKE_INSTALL_PREFIX}/lib" isSystemDir) + if("${isSystemDir}" STREQUAL "-1") + set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") + endif("${isSystemDir}" STREQUAL "-1") +elseif(DEFAULT_RPATH) + # use, i.e. don't skip the full RPATH for the build tree + set(CMAKE_SKIP_BUILD_RPATH FALSE) + + # when building, don't use the install RPATH already + # (but later on when installing) + set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) + + # the RPATH to be used when installing + set(CMAKE_INSTALL_RPATH "") + + # don't add the automatically determined parts of the RPATH + # which point to directories outside the build tree to the install RPATH + set(CMAKE_INSTALL_RPATH_USE_LINK_PATH FALSE) +endif() + include(CheckSymbolExists) check_symbol_exists(sys_signame signal.h HAVE_SIGNAME) @@ -70,10 +110,11 @@ configure_file( # so we are using list as input until we move to new version # TODO once we add build types, we should also set flags such as "-O2 " add_definitions(-D_GNU_SOURCE -D_FILE_OFFSET_BITS=64) +#add_definitions(-D_GNU_SOURCE ) add_definitions() set(CFLAGS_LIST "-std=c11 " - "-ggdb3 -O2 " + "-ggdb3 -O0 " "-Wall -Wshadow -Winline " "-Wstrict-prototypes -Wmissing-prototypes " "-Wmissing-declarations -Wredundant-decls " @@ -99,6 +140,7 @@ add_subdirectory(${CCOMMON_SOURCE_DIR} ${PROJECT_BINARY_DIR}/ccommon) # other dependencies include(FindPackageHandleStandardArgs) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PROJECT_SOURCE_DIR}/cmake") + find_package(Check) if(NOT CHECK_FOUND) message(WARNING "Check is required to build and run tests") @@ -112,14 +154,15 @@ endif(CHECK_FOUND) find_package(Threads) +enable_language(Rust) +include(CMakeCargo) + #set(CMAKE_INCLUDE_CURRENT_DIR) include_directories(${include_directories} "${PROJECT_BINARY_DIR}" "${CCOMMON_SOURCE_DIR}/include" "${PROJECT_SOURCE_DIR}/src") -add_subdirectory("cdb.rs") - # server & (cli) client add_subdirectory(src) @@ -140,5 +183,12 @@ message(STATUS "HAVE_SIGNAME: " ${HAVE_SIGNAME}) message(STATUS "HAVE_BACKTRACE: " ${HAVE_BACKTRACE}) message(STATUS "HAVE_BIG_ENDIAN: " ${HAVE_BIG_ENDIAN}) +if(DUMP_ALL) + get_cmake_property(_variableNames VARIABLES) + foreach (_variableName ${_variableNames}) + message(STATUS "${_variableName}=${${_variableName}}") + endforeach() +endif() + # Note: to uninstall targets, run: # xargs rm < install_manifest.txt diff --git a/cmake/CMakeCargo.cmake b/cmake/CMakeCargo.cmake new file mode 100644 index 000000000..975b0ccd2 --- /dev/null +++ b/cmake/CMakeCargo.cmake @@ -0,0 +1,97 @@ +function(cargo_set_lib_target LIB_NAME) + if(WIN32) + if(CMAKE_SIZEOF_VOID_P EQUAL 8) + set(LIB_TARGET "x86_64-pc-windows-msvc" PARENT_SCOPE) + else() + set(LIB_TARGET "i686-pc-windows-msvc" PARENT_SCOPE) + endif() + elseif(ANDROID) + if(ANDROID_SYSROOT_ABI STREQUAL "x86") + set(LIB_TARGET "i686-linux-android" PARENT_SCOPE) + elseif(ANDROID_SYSROOT_ABI STREQUAL "x86_64") + set(LIB_TARGET "x86_64-linux-android" PARENT_SCOPE) + elseif(ANDROID_SYSROOT_ABI STREQUAL "arm") + set(LIB_TARGET "arm-linux-androideabi" PARENT_SCOPE) + elseif(ANDROID_SYSROOT_ABI STREQUAL "arm64") + set(LIB_TARGET "aarch64-linux-android" PARENT_SCOPE) + endif() + elseif(IOS) + set(LIB_TARGET "universal" PARENT_SCOPE) + elseif(CMAKE_SYSTEM_NAME STREQUAL Darwin) + set(LIB_TARGET "x86_64-apple-darwin" PARENT_SCOPE) + else() + if(RUST_USE_MUSL) + set(RUST_LIBC_NAME "musl") + else() + set(RUST_LIBC_NAME "gnu") + endif() + + if(CMAKE_SIZEOF_VOID_P EQUAL 8) + set(LIB_TARGET "x86_64-unknown-linux-${RUST_LIBC_NAME}" PARENT_SCOPE) + else() + set(LIB_TARGET "i686-unknown-linux-${RUST_LIBC_NAME}" PARENT_SCOPE) + endif() + endif() +endfunction() + +function(cargo_build) + cmake_parse_arguments(CARGO "" "NAME" "" ${ARGN}) + string(REPLACE "-" "_" LIB_NAME ${CARGO_NAME}) + + get_filename_component(CARGO_TARGET_DIR "${CMAKE_CURRENT_BINARY_DIR}" ABSOLUTE) + + cargo_set_lib_target(LIB_NAME) + + if(NOT CMAKE_BUILD_TYPE) + set(LIB_BUILD_TYPE "debug") + elseif(${CMAKE_BUILD_TYPE} STREQUAL "Release") + set(LIB_BUILD_TYPE "release") + else() + set(LIB_BUILD_TYPE "debug") + endif() + + set(SHARED_LIB_FNAME "${CMAKE_SHARED_LIBRARY_PREFIX}${LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}") + set(STATIC_LIB_FNAME "${CMAKE_STATIC_LIBRARY_PREFIX}${LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}") + + set(LIB_BASE_DIR "${CARGO_TARGET_DIR}/${LIB_TARGET}/${LIB_BUILD_TYPE}") + + get_filename_component(SHARED_LIB_FILE "${LIB_BASE_DIR}/${SHARED_LIB_FNAME}" ABSOLUTE) + get_filename_component(STATIC_LIB_FILE "${LIB_BASE_DIR}/${STATIC_LIB_FNAME}" ABSOLUTE) + + if(IOS) + set(CARGO_ARGS "lipo") + else() + set(CARGO_ARGS "build") + list(APPEND CARGO_ARGS "--target" ${LIB_TARGET}) + endif() + + if(${LIB_BUILD_TYPE} STREQUAL "release") + list(APPEND CARGO_ARGS "--release") + endif() + + file(GLOB_RECURSE LIB_SOURCES "*.rs") + + set(CARGO_ENV_COMMAND ${CMAKE_COMMAND} -E env "CARGO_TARGET_DIR=${CARGO_TARGET_DIR}") + + add_custom_command( + OUTPUT ${STATIC_LIB_FILE} + COMMAND ${CARGO_ENV_COMMAND} ${CARGO_EXECUTABLE} ARGS ${CARGO_ARGS} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + DEPENDS ${LIB_SOURCES} + COMMENT "running cargo") + add_custom_target(${CARGO_NAME}_static_target ALL DEPENDS ${STATIC_LIB_FILE}) + add_library(${CARGO_NAME}_static STATIC IMPORTED GLOBAL) + add_dependencies(${CARGO_NAME}_static ${CARGO_NAME}_static_target) + set_target_properties(${CARGO_NAME}_static PROPERTIES IMPORTED_LOCATION ${STATIC_LIB_FILE}) + + add_custom_command( + OUTPUT ${SHARED_LIB_FILE} + COMMAND ${CARGO_ENV_COMMAND} ${CARGO_EXECUTABLE} ARGS ${CARGO_ARGS} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + DEPENDS ${LIB_SOURCES} + COMMENT "running cargo") + add_custom_target(${CARGO_NAME}_shared_target ALL DEPENDS ${SHARED_LIB_FILE}) + add_library(${CARGO_NAME}_shared SHARED IMPORTED GLOBAL) + add_dependencies(${CARGO_NAME}_shared ${CARGO_NAME}_shared_target) + set_target_properties(${CARGO_NAME}_shared PROPERTIES IMPORTED_LOCATION ${SHARED_LIB_FILE}) +endfunction() diff --git a/cmake/CMakeDetermineRustCompiler.cmake b/cmake/CMakeDetermineRustCompiler.cmake new file mode 100644 index 000000000..33a459c0a --- /dev/null +++ b/cmake/CMakeDetermineRustCompiler.cmake @@ -0,0 +1,25 @@ + +if(NOT CMAKE_Rust_COMPILER) + find_package(Rust) + if(RUST_FOUND) + set(CMAKE_Rust_COMPILER "${RUSTC_EXECUTABLE}") + set(CMAKE_Rust_COMPILER_ID "Rust") + set(CMAKE_Rust_COMPILER_VERSION "${RUST_VERSION}") + set(CMAKE_Rust_PLATFORM_ID "Rust") + endif() +endif() + +message(STATUS "Cargo Prefix: ${CARGO_PREFIX}") +message(STATUS "Rust Compiler Version: ${RUSTC_VERSION}") + +mark_as_advanced(CMAKE_Rust_COMPILER) + +if(CMAKE_Rust_COMPILER) + set(CMAKE_Rust_COMPILER_LOADED 1) +endif(CMAKE_Rust_COMPILER) + +configure_file(${CMAKE_SOURCE_DIR}/cmake/CMakeRustCompiler.cmake.in + ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/${CMAKE_VERSION}/CMakeRustCompiler.cmake IMMEDIATE @ONLY) + +set(CMAKE_Rust_COMPILER_ENV_VAR "RUSTC") + diff --git a/cmake/CMakeRustCompiler.cmake.in b/cmake/CMakeRustCompiler.cmake.in new file mode 100644 index 000000000..5916c1c3d --- /dev/null +++ b/cmake/CMakeRustCompiler.cmake.in @@ -0,0 +1,15 @@ + +set(CMAKE_Rust_COMPILER "@CMAKE_Rust_COMPILER@") +set(CMAKE_Rust_COMPILER_ID "@CMAKE_Rust_COMPILER_ID@") +set(CMAKE_Rust_COMPILER_VERSION "@CMAKE_Rust_COMPILER_VERSION@") +set(CMAKE_Rust_COMPILER_LOADED @CMAKE_Rust_COMPILER_LOADED@) +set(CMAKE_Rust_PLATFORM_ID "@CMAKE_Rust_PLATFORM_ID@") + +SET(CMAKE_Rust_SOURCE_FILE_EXTENSIONS rs) +SET(CMAKE_Rust_LINKER_PREFERENCE 40) +#SET(CMAKE_Rust_OUTPUT_EXTENSION_REPLACE 1) +SET(CMAKE_STATIC_LIBRARY_PREFIX_Rust "") +SET(CMAKE_STATIC_LIBRARY_SUFFIX_Rust .a) + +set(CMAKE_Rust_COMPILER_ENV_VAR "RUSTC") + diff --git a/cmake/CMakeRustInformation.cmake b/cmake/CMakeRustInformation.cmake new file mode 100644 index 000000000..05d96c94b --- /dev/null +++ b/cmake/CMakeRustInformation.cmake @@ -0,0 +1,106 @@ + +# +# Usage: rustc [OPTIONS] INPUT +# +# Options: +# -h --help Display this message +# --cfg SPEC Configure the compilation environment +# -L [KIND=]PATH Add a directory to the library search path. The +# optional KIND can be one of dependency, crate, native, +# framework or all (the default). +# -l [KIND=]NAME Link the generated crate(s) to the specified native +# library NAME. The optional KIND can be one of static, +# dylib, or framework. If omitted, dylib is assumed. +# --crate-type [bin|lib|rlib|dylib|cdylib|staticlib|metadata] +# Comma separated list of types of crates for the +# compiler to emit +# --crate-name NAME Specify the name of the crate being built +# --emit [asm|llvm-bc|llvm-ir|obj|link|dep-info] +# Comma separated list of types of output for the +# compiler to emit +# --print [crate-name|file-names|sysroot|cfg|target-list|target-cpus|target-features|relocation-models|code-models] +# Comma separated list of compiler information to print +# on stdout +# -g Equivalent to -C debuginfo=2 +# -O Equivalent to -C opt-level=2 +# -o FILENAME Write output to +# --out-dir DIR Write output to compiler-chosen filename in +# --explain OPT Provide a detailed explanation of an error message +# --test Build a test harness +# --target TARGET Target triple for which the code is compiled +# -W --warn OPT Set lint warnings +# -A --allow OPT Set lint allowed +# -D --deny OPT Set lint denied +# -F --forbid OPT Set lint forbidden +# --cap-lints LEVEL Set the most restrictive lint level. More restrictive +# lints are capped at this level +# -C --codegen OPT[=VALUE] +# Set a codegen option +# -V --version Print version info and exit +# -v --verbose Use verbose output +# +# Additional help: +# -C help Print codegen options +# -W help Print 'lint' options and default settings +# -Z help Print internal options for debugging rustc +# --help -v Print the full set of options rustc accepts +# + +# + +include(CMakeLanguageInformation) + +if(UNIX) + set(CMAKE_Rust_OUTPUT_EXTENSION .o) +else() + set(CMAKE_Rust_OUTPUT_EXTENSION .obj) +endif() + +set(CMAKE_Rust_ECHO_ALL "echo \"TARGET: TARGET_BASE: ") +set(CMAKE_Rust_ECHO_ALL "${CMAKE_Rust_ECHO_ALL} OBJECT: OBJECTS: OBJECT_DIR: SOURCE: SOURCES: ") +set(CMAKE_Rust_ECHO_ALL "${CMAKE_Rust_ECHO_ALL} LINK_LIBRARIES: FLAGS: LINK_FLAGS: \"") + +if(NOT CMAKE_Rust_CREATE_SHARED_LIBRARY) + set(CMAKE_Rust_CREATE_SHARED_LIBRARY + "echo \"CMAKE_Rust_CREATE_SHARED_LIBRARY\"" + "${CMAKE_Rust_ECHO_ALL}" + ) +endif() + +if(NOT CMAKE_Rust_CREATE_SHARED_MODULE) + set(CMAKE_Rust_CREATE_SHARED_MODULE + "echo \"CMAKE_Rust_CREATE_SHARED_MODULE\"" + "${CMAKE_Rust_ECHO_ALL}" + ) +endif() + +if(NOT CMAKE_Rust_CREATE_STATIC_LIBRARY) + set(CMAKE_Rust_CREATE_STATIC_LIBRARY + "echo \"CMAKE_Rust_CREATE_STATIC_LIBRARY\"" + "${CMAKE_Rust_ECHO_ALL}" + ) +endif() + +if(NOT CMAKE_Rust_COMPILE_OBJECT) + set(CMAKE_Rust_COMPILE_OBJECT + "echo \"CMAKE_Rust_COMPILE_OBJECT\"" + "${CMAKE_Rust_ECHO_ALL}" + "${CMAKE_Rust_COMPILER} --emit obj -o ") +endif() + +if(NOT CMAKE_Rust_LINK_EXECUTABLE) + set(CMAKE_Rust_LINK_EXECUTABLE + "echo \"CMAKE_Rust_LINK_EXECUTABLE\"" + "${CMAKE_Rust_ECHO_ALL}" + ) +endif() + +mark_as_advanced( + CMAKE_Rust_FLAGS + CMAKE_Rust_FLAGS_DEBUG + CMAKE_Rust_FLAGS_MINSIZEREL + CMAKE_Rust_FLAGS_RELEASE + CMAKE_Rust_FLAGS_RELWITHDEBINFO) + +set(CMAKE_Rust_INFORMATION_LOADED 1) + diff --git a/cmake/CMakeTestRustCompiler.cmake b/cmake/CMakeTestRustCompiler.cmake new file mode 100644 index 000000000..ff75bb3e0 --- /dev/null +++ b/cmake/CMakeTestRustCompiler.cmake @@ -0,0 +1,3 @@ + +set(CMAKE_Rust_COMPILER_WORKS 1 CACHE INTERNAL "") + diff --git a/cmake/FindRust.cmake b/cmake/FindRust.cmake new file mode 100644 index 000000000..2dbf14ddf --- /dev/null +++ b/cmake/FindRust.cmake @@ -0,0 +1,75 @@ + +set(_CMAKE_FIND_ROOT_PATH_MODE_PROGRAM ${CMAKE_FIND_ROOT_PATH_MODE_PROGRAM}) +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM BOTH) +set(_CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ${CMAKE_FIND_ROOT_PATH_MODE_INCLUDE}) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE BOTH) + +if(WIN32) + set(USER_HOME "$ENV{USERPROFILE}") +else() + set(USER_HOME "$ENV{HOME}") +endif() + +# Find cargo prefix +find_path(CARGO_PREFIX ".cargo" + PATHS "${USER_HOME}") + +if(CARGO_PREFIX MATCHES "NOTFOUND") + message(FATAL_ERROR "Could not find Rust!") +else() + set(CARGO_PREFIX "${CARGO_PREFIX}/.cargo") +endif() + +# Find cargo executable +find_program(CARGO_EXECUTABLE cargo + HINTS "${CARGO_PREFIX}" + PATH_SUFFIXES "bin") +mark_as_advanced(CARGO_EXECUTABLE) + +# Find rustc executable +find_program(RUSTC_EXECUTABLE rustc + HINTS "${CARGO_PREFIX}" + PATH_SUFFIXES "bin") +mark_as_advanced(RUSTC_EXECUTABLE) + +# Find rustdoc executable +find_program(RUSTDOC_EXECUTABLE rustdoc + HINTS "${CARGO_PREFIX}" + PATH_SUFFIXES "bin") +mark_as_advanced(RUSTDOC_EXECUTABLE) + +# Find rust-gdb executable +find_program(RUST_GDB_EXECUTABLE rust-gdb + HINTS "${CARGO_PREFIX}" + PATH_SUFFIXES "bin") +mark_as_advanced(RUST_GDB_EXECUTABLE) + +# Find rust-lldb executable +find_program(RUST_LLDB_EXECUTABLE rust-lldb + HINTS "${CARGO_PREFIX}" + PATH_SUFFIXES "bin") +mark_as_advanced(RUST_LLDB_EXECUTABLE) + +# Find rustup executable +find_program(RUSTUP_EXECUTABLE rustup + HINTS "${CARGO_PREFIX}" + PATH_SUFFIXES "bin") +mark_as_advanced(RUSTUP_EXECUTABLE) + +set(RUST_FOUND FALSE CACHE INTERNAL "") + +if(CARGO_EXECUTABLE AND RUSTC_EXECUTABLE AND RUSTDOC_EXECUTABLE) + set(RUST_FOUND TRUE CACHE INTERNAL "") + + set(CARGO_PREFIX "${CARGO_PREFIX}" CACHE PATH "Rust Cargo prefix") + + execute_process(COMMAND ${RUSTC_EXECUTABLE} --version OUTPUT_VARIABLE RUSTC_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE) + string(REGEX REPLACE "rustc ([^ ]+) .*" "\\1" RUSTC_VERSION "${RUSTC_VERSION}") +endif() + +if(NOT RUST_FOUND) + message(FATAL_ERROR "Could not find Rust!") +endif() + +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM ${_CMAKE_FIND_ROOT_PATH_MODE_PROGRAM}) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ${_CMAKE_FIND_ROOT_PATH_MODE_INCLUDE}) diff --git a/config/cdb.conf b/config/cdb.conf new file mode 100644 index 000000000..6b8c07f71 --- /dev/null +++ b/config/cdb.conf @@ -0,0 +1,17 @@ +# if slab profile is specified, then the profile wil be explicitly set +# otherwise, slab profile will be generated by using growth factor, slab size, +# and item min/max size +# For example, you can specify all item sizes allowed with slab_profile: +# slab_profile: 1024 2048 4096 8192 16384 32768 65536 131072 262144 524288 + +debug_log_level: 4 +debug_log_file: cdb.log +debug_log_nbuf: 1048576 + +klog_file: cdb.cmd +klog_backup: cdb.cmd.old +klog_sample: 100 +klog_max: 1073741824 + +# TODO: this is _obviously_ temporary +# cdb_file_path: /Users/jsimms/git/tub/cdb.rs/dict.cdb diff --git a/deps/ccommon/CMakeLists.txt b/deps/ccommon/CMakeLists.txt index b8fb9e764..1dfce90f9 100644 --- a/deps/ccommon/CMakeLists.txt +++ b/deps/ccommon/CMakeLists.txt @@ -93,7 +93,6 @@ set(CMAKE_MACOSX_RPATH 1) set(CFLAGS_LIST "-std=c11 " "-ggdb3 -O2 " - "-lsubunit " "-Wall " "-Wmissing-prototypes -Wmissing-declarations -Wredundant-decls " "-Wunused-function -Wunused-value -Wunused-variable " diff --git a/deps/ccommon/src/CMakeLists.txt b/deps/ccommon/src/CMakeLists.txt index 43828ce17..97673a298 100644 --- a/deps/ccommon/src/CMakeLists.txt +++ b/deps/ccommon/src/CMakeLists.txt @@ -19,13 +19,13 @@ set(SOURCE cc_ring_array.c cc_signal.c) -# targets to build: here we have both static and dynmaic libs +# targets to build: here we have both static and dynamic libs set(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/lib) add_library(${PROJECT_NAME}-static STATIC ${SOURCE}) add_library(${PROJECT_NAME}-shared SHARED ${SOURCE}) if (OS_PLATFORM STREQUAL "OS_LINUX") - target_link_libraries(${PROJECT_NAME}-static rt) - target_link_libraries(${PROJECT_NAME}-shared rt) + target_link_libraries(${PROJECT_NAME}-static rt subunit m pthread) + target_link_libraries(${PROJECT_NAME}-shared rt subunit m pthread) endif(OS_PLATFORM STREQUAL "OS_LINUX") set_target_properties(${PROJECT_NAME}-static PROPERTIES diff --git a/deps/ccommon/src/cc_debug.c b/deps/ccommon/src/cc_debug.c index 7c766260b..8a61da3e6 100644 --- a/deps/ccommon/src/cc_debug.c +++ b/deps/ccommon/src/cc_debug.c @@ -144,10 +144,10 @@ debug_setup(debug_options_st *options) } /* some adjustment on signal handling */ - if (signal_override(SIGSEGV, "printing stacktrace when segfault", 0, 0, - _stacktrace) < 0) { - goto error; - } +// if (signal_override(SIGSEGV, "printing stacktrace when segfault", 0, 0, +// _stacktrace) < 0) { +// goto error; +// } /* to allow logrotate with nocopytruncate, use SIGHUP to reopen log */ if (signal_override(SIGHUP, "reopen log file", 0, 0, _logrotate) < 0) { diff --git a/deps/ccommon/test/array/CMakeLists.txt b/deps/ccommon/test/array/CMakeLists.txt index 41aa3b488..dc1502b33 100644 --- a/deps/ccommon/test/array/CMakeLists.txt +++ b/deps/ccommon/test/array/CMakeLists.txt @@ -4,7 +4,7 @@ set(test_name check_${suite}) set(source check_${suite}.c) add_executable(${test_name} ${source}) -target_link_libraries(${test_name} ccommon-static ${CHECK_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} m) +target_link_libraries(${test_name} ccommon-static ${CHECK_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} subunit m) add_dependencies(check ${test_name}) add_test(${test_name} ${test_name}) diff --git a/src/server/CMakeLists.txt b/src/server/CMakeLists.txt index fd7805623..032941ea0 100644 --- a/src/server/CMakeLists.txt +++ b/src/server/CMakeLists.txt @@ -17,3 +17,7 @@ endif() if(TARGET_TWEMCACHE) add_subdirectory(twemcache) endif() + +if(TARGET_CDB) + add_subdirectory(cdb) +endif() diff --git a/src/server/cdb/CMakeLists.txt b/src/server/cdb/CMakeLists.txt new file mode 100644 index 000000000..0e9ee1fa6 --- /dev/null +++ b/src/server/cdb/CMakeLists.txt @@ -0,0 +1,32 @@ +add_subdirectory(admin) +add_subdirectory(data) + +set(SOURCE + ${SOURCE} + main.c + setting.c + stats.c) + +set(MODULES + cdb_ffi_static + core + protocol_admin + protocol_memcache + slab + time + util) + +set(LIBS + ccommon-static + dl + ${CMAKE_THREAD_LIBS_INIT}) + +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/_bin) +set(TARGET_NAME ${PROJECT_NAME}_cdb) + +add_executable(${TARGET_NAME} ${SOURCE}) +target_link_libraries(${TARGET_NAME} ${MODULES} ${LIBS}) +add_dependencies(${TARGET_NAME} cdb_ffi_static) + +install(TARGETS ${TARGET_NAME} RUNTIME DESTINATION bin) +add_dependencies(service ${TARGET_NAME}) diff --git a/src/server/cdb/admin/CMakeLists.txt b/src/server/cdb/admin/CMakeLists.txt new file mode 100644 index 000000000..31a7e65f3 --- /dev/null +++ b/src/server/cdb/admin/CMakeLists.txt @@ -0,0 +1,4 @@ +set(SOURCE + ${SOURCE} + ${CMAKE_CURRENT_SOURCE_DIR}/process.c + PARENT_SCOPE) diff --git a/src/server/cdb/admin/process.c b/src/server/cdb/admin/process.c new file mode 100644 index 000000000..a5bd7f4c2 --- /dev/null +++ b/src/server/cdb/admin/process.c @@ -0,0 +1,113 @@ +#include "process.h" + +#include "protocol/admin/admin_include.h" +#include "storage/slab/slab.h" +#include "util/procinfo.h" + +#include +#include + +#define CDB_ADMIN_MODULE_NAME "cdb::admin" + +#define PERSLAB_PREFIX_FMT "CLASS %u:" +#define PERSLAB_METRIC_FMT " %s %s" + +extern struct stats stats; +extern unsigned int nmetric; +static unsigned int nmetric_perslab = METRIC_CARDINALITY(perslab_metrics_st); + +static bool admin_init = false; +static char *buf = NULL; +static size_t cap; + +void +admin_process_setup(void) +{ + log_info("set up the %s module", CDB_ADMIN_MODULE_NAME); + if (admin_init) { + log_warn("%s has already been setup, overwrite", + CDB_ADMIN_MODULE_NAME); + } + + nmetric_perslab = METRIC_CARDINALITY(perslab[0]); + /* perslab metric size <(32 + 20)B, prefix/suffix 12B, total < 64 */ + cap = MAX(nmetric, nmetric_perslab * SLABCLASS_MAX_ID) * METRIC_PRINT_LEN; + buf = cc_alloc(cap); + /* TODO: check return status of cc_alloc */ + + admin_init = true; +} + +void +admin_process_teardown(void) +{ + log_info("tear down the %s module", CDB_ADMIN_MODULE_NAME); + if (!admin_init) { + log_warn("%s has never been setup", CDB_ADMIN_MODULE_NAME); + } + + admin_init = false; +} + +static void +_admin_stats_slab(struct response *rsp, struct request *req) +{ + uint8_t id; + size_t offset = 0; + + for (id = SLABCLASS_MIN_ID; id <= profile_last_id; id++) { + struct metric *metrics = (struct metric *)&perslab[id]; + offset += cc_scnprintf(buf + offset, cap - offset, + PERSLAB_PREFIX_FMT, id); + for (int i = 0; i < nmetric_perslab; i++) { + offset += metric_print(buf + offset, cap - offset, + PERSLAB_METRIC_FMT, &metrics[i]); + } + offset += cc_scnprintf(buf + offset, cap - offset, CRLF); + } + offset += cc_scnprintf(buf + offset, cap - offset, METRIC_END); + + rsp->type = RSP_GENERIC; + rsp->data.data = buf; + rsp->data.len = offset; +} + +static void +_admin_stats_default(struct response *rsp, struct request *req) +{ + procinfo_update(); + rsp->data.data = buf; + rsp->data.len = print_stats(buf, cap, (struct metric *)&stats, nmetric); +} + +static void +_admin_stats(struct response *rsp, struct request *req) +{ + if (bstring_empty(&req->arg)) { + _admin_stats_default(rsp, req); + return; + } + if (req->arg.len == 5 && str5cmp(req->arg.data, ' ', 's', 'l', 'a', 'b')) { + _admin_stats_slab(rsp, req); + } else { + rsp->type = RSP_INVALID; + } +} + +void +admin_process_request(struct response *rsp, struct request *req) +{ + rsp->type = RSP_GENERIC; + + switch (req->type) { + case REQ_STATS: + _admin_stats(rsp, req); + break; + case REQ_VERSION: + rsp->data = str2bstr(VERSION_PRINTED); + break; + default: + rsp->type = RSP_INVALID; + break; + } +} diff --git a/src/server/cdb/admin/process.h b/src/server/cdb/admin/process.h new file mode 100644 index 000000000..361230004 --- /dev/null +++ b/src/server/cdb/admin/process.h @@ -0,0 +1,4 @@ +#pragma once + +void admin_process_setup(void); +void admin_process_teardown(void); diff --git a/src/server/cdb/data/CMakeLists.txt b/src/server/cdb/data/CMakeLists.txt new file mode 100644 index 000000000..31a7e65f3 --- /dev/null +++ b/src/server/cdb/data/CMakeLists.txt @@ -0,0 +1,4 @@ +set(SOURCE + ${SOURCE} + ${CMAKE_CURRENT_SOURCE_DIR}/process.c + PARENT_SCOPE) diff --git a/src/server/cdb/data/process.c b/src/server/cdb/data/process.c new file mode 100644 index 000000000..39d56effb --- /dev/null +++ b/src/server/cdb/data/process.c @@ -0,0 +1,491 @@ +#include "process.h" + +#include "protocol/data/memcache_include.h" +#include "storage/slab/slab.h" +#include "storage/cdb/cdb.h" + +#include +#include +#include + +#define CDB_PROCESS_MODULE_NAME "cdb::process" + +#define OVERSIZE_ERR_MSG "oversized value, cannot be stored" +#define DELTA_ERR_MSG "value is not a number" +#define OOM_ERR_MSG "server is out of memory" +#define CMD_ERR_MSG "command not supported" +#define OTHER_ERR_MSG "unknown server error" + + +typedef enum put_rstatus { + PUT_OK, + PUT_PARTIAL, + PUT_ERROR, +} put_rstatus_t; + +static bool process_init = false; +static process_metrics_st *process_metrics = NULL; +static bool allow_flush = ALLOW_FLUSH; + +static struct CDBHandle *cdb_handle = NULL; + +void +process_setup(process_options_st *options, process_metrics_st *metrics, struct CDBHandle *handle) +{ + log_info("set up the %s module", CDB_PROCESS_MODULE_NAME); + + if (process_init) { + log_warn("%s has already been setup, overwrite", + CDB_PROCESS_MODULE_NAME); + } + + process_metrics = metrics; + + if (options != NULL) { + allow_flush = option_bool(&options->allow_flush); + } + + if (handle == NULL) { + log_crit("handle argument was null! wth?!"); + // TODO(simms): how do i die here? + } + cdb_handle = handle; + + process_init = true; +} + +void +process_teardown(void) +{ + log_info("tear down the %s module", CDB_PROCESS_MODULE_NAME); + if (!process_init) { + log_warn("%s has never been setup", CDB_PROCESS_MODULE_NAME); + } + + if (cdb_handle != NULL) { + struct CDBHandle *p = cdb_handle; + cdb_handle = NULL; + cdb_handle_destroy(p); + } + + allow_flush = false; + process_metrics = NULL; + process_init = false; +} + +static bool +_get_key(struct response *rsp, struct bstring *key) +{ + struct bstring *vstr = cdb_get(cdb_handle, key, &(rsp->vstr)); + + if (vstr != NULL) { + rsp->type = RSP_VALUE; + rsp->key = *key; + rsp->flag = 0; // TODO(simms) FIXME + rsp->vcas = 0; // TODO(simms) FIXME + rsp->vstr.len = vstr->len; + rsp->vstr.data = vstr->data; // TODO(simms) alignment here? + + log_verb("found key at %p, location %p", key, vstr); + return true; + } else { + log_verb("key at %p not found", key); + return false; + } +} + +static void +_process_get(struct response *rsp, struct request *req) +{ + struct bstring *key; + struct response *r = rsp; + uint32_t i; + + INCR(process_metrics, get); + /* use chained responses, move to the next response if key is found. */ + for (i = 0; i < array_nelem(req->keys); ++i) { + INCR(process_metrics, get_key); + key = array_get(req->keys, i); + if (_get_key(r, key)) { + req->nfound++; + r->cas = false; + r = STAILQ_NEXT(r, next); + if (r == NULL) { + INCR(process_metrics, get_ex); + log_warn("get response incomplete due to lack of rsp objects"); + return; + } + INCR(process_metrics, get_key_hit); + } else { + INCR(process_metrics, get_key_miss); + } + } + r->type = RSP_END; + + log_verb("get req %p processed, %d out of %d keys found", req, req->nfound, i); +} + +static void +_process_gets(struct response *rsp, struct request *req) +{ + struct bstring *key; + struct response *r = rsp; + uint32_t i; + + INCR(process_metrics, gets); + /* use chained responses, move to the next response if key is found. */ + for (i = 0; i < array_nelem(req->keys); ++i) { + INCR(process_metrics, gets_key); + key = array_get(req->keys, i); + if (_get_key(r, key)) { + r->cas = true; + r = STAILQ_NEXT(r, next); + if (r == NULL) { + INCR(process_metrics, gets_ex); + log_warn("gets response incomplete due to lack of rsp objects"); + } + req->nfound++; + INCR(process_metrics, gets_key_hit); + } else { + INCR(process_metrics, gets_key_miss); + } + } + r->type = RSP_END; + + log_verb("gets req %p processed, %d out of %d keys found", req, req->nfound, i); +} + +static void +_process_delete(struct response *rsp, struct request *req) +{ + INCR(process_metrics, delete); + rsp->type = RSP_CLIENT_ERROR; + rsp->vstr = str2bstr(CMD_ERR_MSG); + log_verb("delete req %p processed, rsp type %d", req, rsp->type); +} + +/* + * for set/add/replace/cas, we have to recover key from the reserved item, + * because the keys field in the request are only valid for the first segment + * of the request buffer. Once we move to later segments, the areas pointed to + * by these pointers will be overwritten. + */ +static void +_process_set(struct response *rsp, struct request *req) +{ + INCR(process_metrics, set); + INCR(process_metrics, set_ex); + rsp->type = RSP_CLIENT_ERROR; + rsp->vstr = str2bstr(CMD_ERR_MSG); + log_verb("set req %p processed, rsp type %d", req, rsp->type); +} + +static void +_process_add(struct response *rsp, struct request *req) +{ + INCR(process_metrics, add_ex); + rsp->type = RSP_CLIENT_ERROR; + rsp->vstr = str2bstr(CMD_ERR_MSG); + log_verb("add req %p processed, rsp type %d", req, rsp->type); +} + +static void +_process_replace(struct response *rsp, struct request *req) +{ + INCR(process_metrics, replace_ex); + rsp->type = RSP_CLIENT_ERROR; + rsp->vstr = str2bstr(CMD_ERR_MSG); + log_verb("replace req %p processed, rsp type %d", req, rsp->type); +} + +static void +_process_cas(struct response *rsp, struct request *req) +{ + INCR(process_metrics, cas_ex); + rsp->type = RSP_CLIENT_ERROR; + rsp->vstr = str2bstr(CMD_ERR_MSG); + log_verb("cas req %p processed, rsp type %d", req, rsp->type); +} + +static void +_process_incr(struct response *rsp, struct request *req) +{ + INCR(process_metrics, incr); + INCR(process_metrics, incr_ex); + rsp->type = RSP_CLIENT_ERROR; + rsp->vstr = str2bstr(CMD_ERR_MSG); + log_verb("incr req %p processed, rsp type %d", req, rsp->type); +} + +static void +_process_decr(struct response *rsp, struct request *req) +{ + INCR(process_metrics, decr); + INCR(process_metrics, decr_ex); + rsp->type = RSP_CLIENT_ERROR; + rsp->vstr = str2bstr(CMD_ERR_MSG); + log_verb("decr req %p processed, rsp type %d", req, rsp->type); +} + +static void +_process_append(struct response *rsp, struct request *req) +{ + INCR(process_metrics, append_ex); + rsp->type = RSP_CLIENT_ERROR; + rsp->vstr = str2bstr(CMD_ERR_MSG); + log_verb("append req %p processed, rsp type %d", req, rsp->type); +} + +static void +_process_prepend(struct response *rsp, struct request *req) +{ + INCR(process_metrics, prepend_ex); + rsp->type = RSP_CLIENT_ERROR; + rsp->vstr = str2bstr(CMD_ERR_MSG); + log_verb("prepend req %p processed, rsp type %d", req, rsp->type); +} + +static void +_process_flush(struct response *rsp, struct request *req) +{ + INCR(process_metrics, flush); + rsp->type = RSP_CLIENT_ERROR; + rsp->vstr = str2bstr(CMD_ERR_MSG); + log_info("flush req %p processed, rsp type %d", req, rsp->type); +} + +void +process_request(struct response *rsp, struct request *req) +{ + log_verb("processing req %p, write rsp to %p", req, rsp); + INCR(process_metrics, process_req); + + switch (req->type) { + case REQ_GET: + _process_get(rsp, req); + break; + + case REQ_GETS: + _process_gets(rsp, req); + break; + + case REQ_DELETE: + _process_delete(rsp, req); + break; + + case REQ_SET: + _process_set(rsp, req); + break; + + case REQ_ADD: + _process_add(rsp, req); + break; + + case REQ_REPLACE: + _process_replace(rsp, req); + break; + + case REQ_CAS: + _process_cas(rsp, req); + break; + + case REQ_INCR: + _process_incr(rsp, req); + break; + + case REQ_DECR: + _process_decr(rsp, req); + break; + + case REQ_APPEND: + _process_append(rsp, req); + break; + + case REQ_PREPEND: + _process_prepend(rsp, req); + break; + + case REQ_FLUSH: + _process_flush(rsp, req); + break; + + default: + rsp->type = RSP_CLIENT_ERROR; + rsp->vstr = str2bstr(CMD_ERR_MSG); + break; + } +} + +static inline void +_cleanup(struct request *req, struct response *rsp) +{ + struct response *nr = STAILQ_NEXT(rsp, next); + + request_reset(req); + /* return all but the first response */ + if (nr != NULL) { + response_return_all(&nr); + } + + response_reset(rsp); + req->rsp = rsp; +} + +int +cdb_process_read(struct buf **rbuf, struct buf **wbuf, void **data) +{ + parse_rstatus_t status; + struct request *req; /* data should be NULL or hold a req pointer */ + struct response *rsp; + + log_verb("post-read processing"); + + /* deal with the stateful part: request and response */ + req = (*data != NULL) ? *data : request_borrow(); + if (req == NULL) { + /* TODO(yao): simply return for now, better to respond with OOM */ + log_error("cannot process request: OOM"); + INCR(process_metrics, process_ex); + + return -1; + } + rsp = (req->rsp != NULL) ? req->rsp : response_borrow(); + if (rsp == NULL) { + request_return(&req); + /* TODO(yao): simply return for now, better to respond with OOM */ + log_error("cannot process request: OOM"); + INCR(process_metrics, process_ex); + + return -1; + } + + /* keep parse-process-compose until running out of data in rbuf */ + while (buf_rsize(*rbuf) > 0) { + struct response *nr; + int i, card; + + /* stage 1: parsing */ + log_verb("%"PRIu32" bytes left", buf_rsize(*rbuf)); + + status = parse_req(req, *rbuf); + if (status == PARSE_EUNFIN) { + buf_lshift(*rbuf); + return 0; + } + if (status != PARSE_OK) { + /* parsing errors are all client errors, since we don't know + * how to recover from client errors in this condition (we do not + * have a valid request so we don't know where the invalid request + * ends), we should close the connection + */ + log_warn("illegal request received, status: %d", status); + return -1; + } + + if (req->swallow) { /* skip to the end of current request */ + continue; + } + + /* stage 2: processing- check for quit, allocate response(s), process */ + + /* quit is special, no response expected */ + if (req->type == REQ_QUIT) { + log_info("peer called quit"); + return -1; + } + + /* find cardinality of the request and get enough response objects */ + card = array_nelem(req->keys) - 1; /* we already have one in rsp */ + if (req->type == REQ_GET || req->type == REQ_GETS) { + /* extra response object for the "END" line after values */ + card++; + } + for (i = 0, nr = rsp; + i < card; + i++, STAILQ_NEXT(nr, next) = response_borrow(), nr = + STAILQ_NEXT(nr, next)) { + if (nr == NULL) { + log_error("cannot acquire response: OOM"); + INCR(process_metrics, process_ex); + _cleanup(req, rsp); + return -1; + } + } + + /* actual processing */ + process_request(rsp, req); + if (req->partial) { /* implies end of rbuf w/o complete processing */ + /* in this case, do not attempt to log or write response */ + buf_lshift(*rbuf); + return 0; + } + + /* stage 3: write response(s) if necessary */ + + /* noreply means no need to write to buffers */ + card++; + if (!req->noreply) { + nr = rsp; + if (req->type == REQ_GET || req->type == REQ_GETS) { + /* for get/gets, card is determined by number of values */ + card = req->nfound + 1; + } + for (i = 0; i < card; nr = STAILQ_NEXT(nr, next), ++i) { + if (compose_rsp(wbuf, nr) < 0) { + log_error("composing rsp erred"); + INCR(process_metrics, process_ex); + _cleanup(req, rsp); + return -1; + } + } + } + + /* logging, clean-up */ + klog_write(req, rsp); + _cleanup(req, rsp); + } + + return 0; +} + + +int +cdb_process_write(struct buf **rbuf, struct buf **wbuf, void **data) +{ + log_verb("post-write processing"); + + buf_lshift(*rbuf); + dbuf_shrink(rbuf); + buf_lshift(*wbuf); + dbuf_shrink(wbuf); + + return 0; +} + + +int +cdb_process_error(struct buf **rbuf, struct buf **wbuf, void **data) +{ + struct request *req = *data; + struct response *rsp; + + log_verb("post-error processing"); + + /* normalize buffer size */ + buf_reset(*rbuf); + dbuf_shrink(rbuf); + buf_reset(*wbuf); + dbuf_shrink(wbuf); + + /* release request data & associated reserved data */ + if (req != NULL) { + rsp = req->rsp; + if (req->reserved != NULL) { + item_release((struct item **)&req->reserved); + } + request_return(&req); + response_return_all(&rsp); + } + + return 0; +} diff --git a/src/server/cdb/data/process.h b/src/server/cdb/data/process.h new file mode 100644 index 000000000..47f44a659 --- /dev/null +++ b/src/server/cdb/data/process.h @@ -0,0 +1,81 @@ +#pragma once + +#include "storage/cdb/cdb.h" + +#include +#include +#include +#include + +#define ALLOW_FLUSH false + +/* name type default description */ +#define PROCESS_OPTION(ACTION) \ + ACTION( allow_flush, OPTION_TYPE_BOOL, ALLOW_FLUSH, "allow flushing on the data port" ) + +typedef struct { + PROCESS_OPTION(OPTION_DECLARE) +} process_options_st; + +/* name type description */ +#define PROCESS_METRIC(ACTION) \ + ACTION( process_req, METRIC_COUNTER, "# requests processed" )\ + ACTION( process_ex, METRIC_COUNTER, "# processing error" )\ + ACTION( process_server_ex, METRIC_COUNTER, "# internal error" )\ + ACTION( get, METRIC_COUNTER, "# get requests" )\ + ACTION( get_key, METRIC_COUNTER, "# keys by get" )\ + ACTION( get_key_hit, METRIC_COUNTER, "# key hits by get" )\ + ACTION( get_key_miss, METRIC_COUNTER, "# key misses by get" )\ + ACTION( get_ex, METRIC_COUNTER, "# get errors" )\ + ACTION( gets, METRIC_COUNTER, "# gets requests" )\ + ACTION( gets_key, METRIC_COUNTER, "# keys by gets" )\ + ACTION( gets_key_hit, METRIC_COUNTER, "# key hits by gets" )\ + ACTION( gets_key_miss, METRIC_COUNTER, "# key misses by gets" )\ + ACTION( gets_ex, METRIC_COUNTER, "# gets errors" )\ + ACTION( delete, METRIC_COUNTER, "# delete requests" )\ + ACTION( delete_deleted, METRIC_COUNTER, "# delete successes" )\ + ACTION( delete_notfound, METRIC_COUNTER, "# delete not_founds" )\ + ACTION( set, METRIC_COUNTER, "# set requests" )\ + ACTION( set_stored, METRIC_COUNTER, "# set successes" )\ + ACTION( set_ex, METRIC_COUNTER, "# set errors" )\ + ACTION( add, METRIC_COUNTER, "# add requests" )\ + ACTION( add_stored, METRIC_COUNTER, "# add successes" )\ + ACTION( add_notstored, METRIC_COUNTER, "# add failures" )\ + ACTION( add_ex, METRIC_COUNTER, "# add errors" )\ + ACTION( replace, METRIC_COUNTER, "# replace requests" )\ + ACTION( replace_stored, METRIC_COUNTER, "# replace successes" )\ + ACTION( replace_notstored, METRIC_COUNTER, "# replace failures" )\ + ACTION( replace_ex, METRIC_COUNTER, "# replace errors" )\ + ACTION( cas, METRIC_COUNTER, "# cas requests" )\ + ACTION( cas_stored, METRIC_COUNTER, "# cas successes" )\ + ACTION( cas_exists, METRIC_COUNTER, "# cas bad values" )\ + ACTION( cas_notfound, METRIC_COUNTER, "# cas not_founds" )\ + ACTION( cas_ex, METRIC_COUNTER, "# cas errors" )\ + ACTION( incr, METRIC_COUNTER, "# incr requests" )\ + ACTION( incr_stored, METRIC_COUNTER, "# incr successes" )\ + ACTION( incr_notfound, METRIC_COUNTER, "# incr not_founds" )\ + ACTION( incr_ex, METRIC_COUNTER, "# incr errors" )\ + ACTION( decr, METRIC_COUNTER, "# decr requests" )\ + ACTION( decr_stored, METRIC_COUNTER, "# decr successes" )\ + ACTION( decr_notfound, METRIC_COUNTER, "# decr not_founds" )\ + ACTION( decr_ex, METRIC_COUNTER, "# decr errors" )\ + ACTION( append, METRIC_COUNTER, "# append requests" )\ + ACTION( append_stored, METRIC_COUNTER, "# append successes" )\ + ACTION( append_notstored, METRIC_COUNTER, "# append not_founds" )\ + ACTION( append_ex, METRIC_COUNTER, "# append errors" )\ + ACTION( prepend, METRIC_COUNTER, "# prepend requests" )\ + ACTION( prepend_stored, METRIC_COUNTER, "# prepend successes" )\ + ACTION( prepend_notstored, METRIC_COUNTER, "# prepend not_founds" )\ + ACTION( prepend_ex, METRIC_COUNTER, "# prepend errors" )\ + ACTION( flush, METRIC_COUNTER, "# flush_all requests" ) + +typedef struct { + PROCESS_METRIC(METRIC_DECLARE) +} process_metrics_st; + +void process_setup(process_options_st *options, process_metrics_st *metrics, struct CDBHandle *cdb_handle); +void process_teardown(void); + +int cdb_process_read(struct buf **rbuf, struct buf **wbuf, void **data); +int cdb_process_write(struct buf **rbuf, struct buf **wbuf, void **data); +int cdb_process_error(struct buf **rbuf, struct buf **wbuf, void **data); diff --git a/src/server/cdb/main.c b/src/server/cdb/main.c new file mode 100644 index 000000000..838e833ca --- /dev/null +++ b/src/server/cdb/main.c @@ -0,0 +1,235 @@ +#include "admin/process.h" +#include "setting.h" +#include "stats.h" +#include "storage/cdb/cdb.h" + +#include "time/time.h" +#include "util/util.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +struct data_processor worker_processor = { + cdb_process_read, + cdb_process_write, + cdb_process_error, +}; + +static void +show_usage(void) +{ + log_stdout( + "Usage:" CRLF + " pelikan_cdb [option|config]" CRLF + ); + log_stdout( + "Description:" CRLF + " pelikan_cdb is one of the unified cache backends. " CRLF + " It uses a slab-based storage to cache key/val pairs. " CRLF + " It speaks the memcached ASCII protocol and supports almost " CRLF + " all ASCII memcached commands." CRLF + ); + log_stdout( + "Command-line options:" CRLF + " -h, --help show this message" CRLF + " -v, --version show version number" CRLF + " -c, --config list & describe all options in config" CRLF + " -s, --stats list & describe all metrics in stats" CRLF + ); + log_stdout( + "Example:" CRLF + " pelikan_cdb cdb.conf" CRLF CRLF + "Sample config files can be found under the config dir." CRLF + ); +} + +static void +teardown(void) +{ + core_worker_teardown(); + core_server_teardown(); + core_admin_teardown(); + admin_process_teardown(); + process_teardown(); + cdb_teardown(); + klog_teardown(); + compose_teardown(); + parse_teardown(); + response_teardown(); + request_teardown(); + procinfo_teardown(); + time_teardown(); + + timing_wheel_teardown(); + tcp_teardown(); + sockio_teardown(); + event_teardown(); + dbuf_teardown(); + buf_teardown(); + + debug_teardown(); + log_teardown(); +} + +static struct CDBHandle* +setup_cdb_handle(void) +{ + cdb_setup(); + + char *cdb_file_path = strdup("/Users/jsimms/pelikan/dict.cdb"); + return cdb_handle_create(cdb_file_path); +} + +static void +setup(void) +{ + char *fname = NULL; + uint64_t intvl; + + + if (atexit(teardown) != 0) { + log_stderr("cannot register teardown procedure with atexit()"); + exit(EX_OSERR); /* only failure comes from NOMEM */ + } + + /* Setup logging first */ + log_setup(&stats.log); + if (debug_setup(&setting.debug) != CC_OK) { + log_stderr("debug log setup failed"); + exit(EX_CONFIG); + } + + /* setup top-level application options */ + if (option_bool(&setting.cdb.daemonize)) { + daemonize(); + } + fname = option_str(&setting.cdb.pid_filename); + if (fname != NULL) { + /* to get the correct pid, call create_pidfile after daemonize */ + create_pidfile(fname); + } + + + /* setup library modules */ + buf_setup(&setting.buf, &stats.buf); + dbuf_setup(&setting.dbuf, &stats.dbuf); + event_setup(&stats.event); + sockio_setup(&setting.sockio, &stats.sockio); + tcp_setup(&setting.tcp, &stats.tcp); + timing_wheel_setup(&stats.timing_wheel); + + /* setup pelikan modules */ + time_setup(); + procinfo_setup(&stats.procinfo); + request_setup(&setting.request, &stats.request); + response_setup(&setting.response, &stats.response); + parse_setup(&stats.parse_req, NULL); + compose_setup(NULL, &stats.compose_rsp); + klog_setup(&setting.klog, &stats.klog); + + struct CDBHandle* cdb_handle = setup_cdb_handle(); + if (cdb_handle == NULL) { + log_stderr("failed to set up cdb"); + goto error; + } + + process_setup(&setting.process, &stats.process, cdb_handle); + admin_process_setup(); + core_admin_setup(&setting.admin); + core_server_setup(&setting.server, &stats.server); + core_worker_setup(&setting.worker, &stats.worker); + + /* adding recurring events to maintenance/admin thread */ + intvl = option_uint(&setting.cdb.dlog_intvl); + if (core_admin_register(intvl, debug_log_flush, NULL) == NULL) { + log_stderr("Could not register timed event to flush debug log"); + goto error; + } + + intvl = option_uint(&setting.cdb.klog_intvl); + if (core_admin_register(intvl, klog_flush, NULL) == NULL) { + log_error("Could not register timed event to flush command log"); + goto error; + } + + return; + +error: + if (fname != NULL) { + remove_pidfile(fname); + } + + /* since we registered teardown with atexit, it'll be called upon exit */ + exit(EX_CONFIG); +} + +int +main(int argc, char **argv) +{ + rstatus_i status = CC_OK;; + FILE *fp = NULL; + + if (argc > 2) { + show_usage(); + exit(EX_USAGE); + } + + if (argc == 1) { + log_stderr("launching server with default values."); + } else { + /* argc == 2 */ + if (strcmp(argv[1], "-h") == 0 || strcmp(argv[1], "--help") == 0) { + show_usage(); + exit(EX_OK); + } + if (strcmp(argv[1], "-v") == 0 || strcmp(argv[1], "--version") == 0) { + show_version(); + exit(EX_OK); + } + if (strcmp(argv[1], "-c") == 0 || strcmp(argv[1], "--config") == 0) { + option_describe_all((struct option *)&setting, nopt); + exit(EX_OK); + } + if (strcmp(argv[1], "-s") == 0 || strcmp(argv[1], "--stats") == 0) { + metric_describe_all((struct metric *)&stats, nmetric); + exit(EX_OK); + } + fp = fopen(argv[1], "r"); + if (fp == NULL) { + log_stderr("cannot open config: incorrect path or doesn't exist"); + exit(EX_DATAERR); + } + } + + if (option_load_default((struct option *)&setting, nopt) != CC_OK) { + log_stderr("failed to load default option values"); + exit(EX_CONFIG); + } + + if (fp != NULL) { + log_stderr("load config from %s", argv[1]); + status = option_load_file(fp, (struct option *)&setting, nopt); + fclose(fp); + } + if (status != CC_OK) { + log_stderr("failed to load config"); + exit(EX_DATAERR); + } + + setup(); + option_print_all((struct option *)&setting, nopt); + + core_run(&worker_processor); + + exit(EX_OK); +} diff --git a/src/server/cdb/setting.c b/src/server/cdb/setting.c new file mode 100644 index 000000000..7ca76fdda --- /dev/null +++ b/src/server/cdb/setting.c @@ -0,0 +1,21 @@ +#include "setting.h" + +struct setting setting = { + { CDB_OPTION(OPTION_INIT) }, + { ADMIN_OPTION(OPTION_INIT) }, + { SERVER_OPTION(OPTION_INIT) }, + { WORKER_OPTION(OPTION_INIT) }, + { PROCESS_OPTION(OPTION_INIT) }, + { KLOG_OPTION(OPTION_INIT) }, + { REQUEST_OPTION(OPTION_INIT) }, + { RESPONSE_OPTION(OPTION_INIT) }, + { SLAB_OPTION(OPTION_INIT) }, + { ARRAY_OPTION(OPTION_INIT) }, + { BUF_OPTION(OPTION_INIT) }, + { DBUF_OPTION(OPTION_INIT) }, + { DEBUG_OPTION(OPTION_INIT) }, + { SOCKIO_OPTION(OPTION_INIT) }, + { TCP_OPTION(OPTION_INIT) }, +}; + +unsigned int nopt = OPTION_CARDINALITY(struct setting); diff --git a/src/server/cdb/setting.h b/src/server/cdb/setting.h new file mode 100644 index 000000000..fff0d46e0 --- /dev/null +++ b/src/server/cdb/setting.h @@ -0,0 +1,53 @@ +#pragma once + +#include "data/process.h" + +#include "core/core.h" +#include "protocol/data/memcache_include.h" +#include "storage/slab/item.h" +#include "storage/slab/slab.h" + +#include +#include +#include +#include +#include +#include +#include + +/* option related */ +/* name type default description */ +#define CDB_OPTION(ACTION) \ + ACTION( daemonize, OPTION_TYPE_BOOL, false, "daemonize the process" )\ + ACTION( pid_filename, OPTION_TYPE_STR, NULL, "file storing the pid" )\ + ACTION( cdb_file_path, OPTION_TYPE_STR, "cdb.cdb", "location of the .cdb file" )\ + ACTION( dlog_intvl, OPTION_TYPE_UINT, 500, "debug log flush interval(ms)" )\ + ACTION( klog_intvl, OPTION_TYPE_UINT, 100, "cmd log flush interval(ms)" ) + +typedef struct { + CDB_OPTION(OPTION_DECLARE) +} cdb_options_st; + +struct setting { + /* top-level */ + cdb_options_st cdb; + /* application modules */ + admin_options_st admin; + server_options_st server; + worker_options_st worker; + process_options_st process; + klog_options_st klog; + request_options_st request; + response_options_st response; + slab_options_st slab; + /* ccommon libraries */ + array_options_st array; + buf_options_st buf; + dbuf_options_st dbuf; + debug_options_st debug; + sockio_options_st sockio; + tcp_options_st tcp; +}; + +extern struct setting setting; +extern unsigned int nopt; diff --git a/src/server/cdb/stats.c b/src/server/cdb/stats.c new file mode 100644 index 000000000..3af172cba --- /dev/null +++ b/src/server/cdb/stats.c @@ -0,0 +1,23 @@ +#include "stats.h" + +struct stats stats = { + { PROCINFO_METRIC(METRIC_INIT) }, + { PROCESS_METRIC(METRIC_INIT) }, + { PARSE_REQ_METRIC(METRIC_INIT) }, + { COMPOSE_RSP_METRIC(METRIC_INIT) }, + { KLOG_METRIC(METRIC_INIT) }, + { REQUEST_METRIC(METRIC_INIT) }, + { RESPONSE_METRIC(METRIC_INIT) }, + { SLAB_METRIC(METRIC_INIT) }, + { CORE_SERVER_METRIC(METRIC_INIT) }, + { CORE_WORKER_METRIC(METRIC_INIT) }, + { BUF_METRIC(METRIC_INIT) }, + { DBUF_METRIC(METRIC_INIT) }, + { EVENT_METRIC(METRIC_INIT) }, + { LOG_METRIC(METRIC_INIT) }, + { SOCKIO_METRIC(METRIC_INIT) }, + { TCP_METRIC(METRIC_INIT) }, + { TIMING_WHEEL_METRIC(METRIC_INIT) }, +}; + +unsigned int nmetric = METRIC_CARDINALITY(stats); diff --git a/src/server/cdb/stats.h b/src/server/cdb/stats.h new file mode 100644 index 000000000..2dd161543 --- /dev/null +++ b/src/server/cdb/stats.h @@ -0,0 +1,41 @@ +#pragma once + +#include "data/process.h" + +#include "core/core.h" +#include "protocol/data/memcache_include.h" +#include "storage/slab/item.h" +#include "storage/slab/slab.h" +#include "util/procinfo.h" + +#include +#include +#include +#include +#include