diff --git a/.travis.yml b/.travis.yml index 41c2b344e..26a944005 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,12 @@ sudo: false language: c +dist: xenial # using anchor to import sources into linux builds addons: apt: &apt sources: - ubuntu-toolchain-r-test - - llvm-toolchain-precise-3.6 - - llvm-toolchain-precise-3.7 - - llvm-toolchain-precise # important for allowed-to-fail matching # see https://docs.travis-ci.com/user/customizing-the-build#Rows-that-are-Allowed-to-Fail @@ -18,115 +16,34 @@ env: # travis currently does not support directly setting gcc/clang with versions # (e.g. gcc-4.8) as value for the compiler key. So we will have to manually # request these packages and use environment varibles to create the matrix. -# -# In the case of osx, use brew to install the paritcular versions, instead of -# specifying with packages. matrix: include: - # gcc 4.8 on linux - - env: - - C_COMPILER=gcc-4.8 + - name: "gcc-5 on Linux" + compiler: gcc addons: apt: <<: *apt packages: - - gcc-4.8 - libsubunit-dev - - # gcc 4.9 on linux - - env: - - C_COMPILER=gcc-4.9 - addons: - apt: - <<: *apt - packages: - - gcc-4.9 - - libsubunit-dev - - # gcc 5 on linux - - env: - - C_COMPILER=gcc-5 - addons: - apt: - <<: *apt - packages: - - gcc-5 - - libsubunit-dev - - # gcc 5 on linux - - env: - - C_COMPILER=gcc-5 + - name: "gcc-5 on Linux, Rust enabled" + compiler: gcc + env: - RUST_ENABLED=1 addons: apt: <<: *apt packages: - - gcc-5 - libsubunit-dev - # clang 3.6 on linux - - env: - - C_COMPILER=clang-3.6 - addons: - apt: - <<: *apt - packages: - - clang-3.6 - - libsubunit-dev - - # clang 3.7 on linux - - env: - - C_COMPILER=clang-3.7 - addons: - apt: - <<: *apt - packages: - - clang-3.7 - - libsubunit-dev - - ## gcc 4.8 on osx - #- os: osx - # env: FORMULA=gcc48 COMPILER=gcc C_COMPILER=gcc-4.8 - # - ## gcc 4.9 on osx - #- os: osx - # env: FORMULA=gcc49 COMPILER=gcc C_COMPILER=gcc-4.9 - # - ## gcc 5 on osx - #- os: osx - # env: FORMULA=gcc5 COMPILER=gcc C_COMPILER=gcc-5 - - # OSX 10.13 - # Apple LLVM version 9.1.0 (clang-902.0.39.2) - # Target: x86_64-apple-darwin17.6.0 - os: osx - osx_image: xcode9.4 - env: - - C_COMPILER=clang - - ALLOWED_TO_FAIL=1 - - # OSX 10.12 - # Apple LLVM version 9.0.0 (clang-900.0.39.2) - # Target: x86_64-apple-darwin16.7.0 - - os: osx - osx_image: xcode9.2 - env: - - C_COMPILER=clang - - ALLOWED_TO_FAIL=1 + osx_image: xcode10.1 + compiler: clang allow_failures: - os: osx - osx_image: xcode9.4 - env: - - C_COMPILER=clang - - ALLOWED_TO_FAIL=1 - - - os: osx - osx_image: xcode9.2 - env: - - C_COMPILER=clang - - ALLOWED_TO_FAIL=1 + osx_image: xcode10.1 + compiler: clang before_install: - ./ci/before-install.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 2d2a2c05a..afff55bc1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,9 @@ -cmake_minimum_required(VERSION 2.6) +cmake_minimum_required(VERSION 2.8) project(ccommon C) +# Uncomment the following to output dependency graph debugging messages +# set_property(GLOBAL PROPERTY GLOBAL_DEPENDS_DEBUG_MODE 1) + enable_testing() ################### @@ -36,8 +39,8 @@ endif() # config.h.in has to include entries set/tested here for them to have effect # version info -set(${PROJECT_NAME}_VERSION_MAJOR 1) -set(${PROJECT_NAME}_VERSION_MINOR 2) +set(${PROJECT_NAME}_VERSION_MAJOR 2) +set(${PROJECT_NAME}_VERSION_MINOR 1) set(${PROJECT_NAME}_VERSION_PATCH 0) set(${PROJECT_NAME}_VERSION ${${PROJECT_NAME}_VERSION_MAJOR}.${${PROJECT_NAME}_VERSION_MINOR}.${${PROJECT_NAME}_VERSION_PATCH} @@ -54,6 +57,7 @@ option(HAVE_STATS "stats enabled by default" ON) option(HAVE_DEBUG_MM "debugging oriented memory management disabled by default" OFF) option(COVERAGE "code coverage" OFF) option(HAVE_RUST "rust bindings not built by default" OFF) +option(HAVE_ITT_INSTRUMENTATION "instrument code with ITT API" OFF) if(HAVE_RUST) option(RUST_VERBOSE_BUILD "pass -vv to cargo compilation" OFF) @@ -96,9 +100,7 @@ check_symbol_exists(sys_signame signal.h HAVE_SIGNAME) include(CheckFunctionExists) check_function_exists(backtrace HAVE_BACKTRACE) - -include(TestBigEndian) -test_big_endian(HAVE_BIG_ENDIAN) +check_function_exists(accept4 HAVE_ACCEPT4) # how to use config.h.in to generate config.h # this has to be set _after_ the above checks @@ -114,12 +116,23 @@ configure_file( # set compiler flags # string concat is easier in 3.0, but older versions don't have the concat subcommand # so we are using list as input until we move to new version -# TODO add build types add_definitions(-D_GNU_SOURCE -D_FILE_OFFSET_BITS=64) +# Set a default build type (Release) if none was specified + +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif() + +if(CMAKE_BUILD_TYPE MATCHES Debug) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O0") +else() + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2") +endif() + set(CMAKE_MACOSX_RPATH 1) set(CFLAGS_LIST "-std=c11 " - "-ggdb3 -O2 " + "-ggdb3 " "-Wall " "-Wmissing-prototypes -Wmissing-declarations -Wredundant-decls " "-Wunused-function -Wunused-value -Wunused-variable " @@ -132,7 +145,10 @@ if(CMAKE_COMPILER_IS_GNUCC) endif() if (COVERAGE) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -Wall -W -fprofile-arcs -ftest-coverage") + if(NOT ${CMAKE_BUILD_TYPE} MATCHES Debug) + message(WARNING "Code coverage results with an optimised (non-Debug) build may be misleading" ) + endif() + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage") endif(COVERAGE) # test dependencies @@ -150,6 +166,17 @@ if(CHECK_FOUND) endif(NOT CHECK_WORKING) endif(CHECK_FOUND) +if (HAVE_ITT_INSTRUMENTATION) + if(PKG_CONFIG_FOUND) + pkg_check_modules(ITTNOTIFY REQUIRED ittnotify>=1.0) + else() + find_package(ITTNOTIFY REQUIRED 1.0) + endif() + include_directories(${ITTNOTIFY_INCLUDE_DIRS}) + link_directories(${ITTNOTIFY_LIBRARY_DIRS}) + link_libraries(${ITTNOTIFY_LIBRARIES}) +endif(HAVE_ITT_INSTRUMENTATION) + find_package(Threads) @@ -159,28 +186,32 @@ include_directories( "${PROJECT_BINARY_DIR}" "include") -if(HAVE_RUST) - enable_language(Rust) - include(CMakeCargo) - add_subdirectory(rust) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DHAVE_RUST=1") -endif() - - ################### # things to build # ################### add_subdirectory(src) + if(CHECK_WORKING) include_directories(${include_directories} "${CHECK_INCLUDES}") add_subdirectory(test) endif(CHECK_WORKING) + +if(HAVE_RUST) + enable_language(Rust) + include(CMakeCargo) + add_subdirectory(rust) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DHAVE_RUST=1") +endif() + + ################### # print a summary # ################### +message(STATUS "CMAKE_BUILD_TYPE: " ${CMAKE_BUILD_TYPE}) + message(STATUS "PLATFORM: " ${OS_PLATFORM}) message(STATUS "CPPFLAGS: " ${CMAKE_CPP_FLAGS}) @@ -189,6 +220,5 @@ message(STATUS "CFLAGS: " ${CMAKE_C_FLAGS}) message(STATUS "HAVE_SIGNAME: " ${HAVE_SIGNAME}) message(STATUS "HAVE_BACKTRACE: " ${HAVE_BACKTRACE}) -message(STATUS "HAVE_BIG_ENDIAN: " ${HAVE_BIG_ENDIAN}) message(STATUS "CHECK_WORKING: " ${CHECK_WORKING}) diff --git a/LICENSE b/LICENSE index ecc06d346..5c3aef4f8 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright 2013-2015 Twitter, Inc +Copyright 2013-2018 Twitter, Inc Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/NOTICE b/NOTICE index 2b7bb058e..e0672064d 100644 --- a/NOTICE +++ b/NOTICE @@ -22,3 +22,5 @@ * Boston, MA 02111-1307, USA. */ +We use the CMakeRust project (https://github.com/Devolutions/CMakeRust) under +the Apache 2.0 License. diff --git a/ci/before-install.sh b/ci/before-install.sh index cff9327c3..b5f49421a 100755 --- a/ci/before-install.sh +++ b/ci/before-install.sh @@ -11,21 +11,6 @@ trap cleanup EXIT TOPLEVEL="$(git -C "$(cd "$(dirname "$0")" >/dev/null || exit 1; pwd)" rev-parse --show-toplevel)" || die 'failed to find TOPLEVEL' -# for osx: 0. update brew; 1. install cmake if missing; 2. (gcc) unlink pre-installed gcc; 3. (gcc) install desired version of gcc - -if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - brew update &>/dev/null - brew install cmake || true # xcode 8.1 is missing cmake - - if [[ "$C_COMPILER" =~ ^gcc && -n "${FORMULA:-}" ]]; then - brew unlink gcc || true - brew unlink "$FORMULA" || true - brew install "$FORMULA" - fi -fi - -export CC="$C_COMPILER" - if [[ -n "${RUST_ENABLED:-}" ]]; then curl https://sh.rustup.rs -sSf | sh -s -- -y fi diff --git a/ci/install-check.sh b/ci/install-check.sh index 628d6d229..95d17c9e0 100755 --- a/ci/install-check.sh +++ b/ci/install-check.sh @@ -23,12 +23,28 @@ CHECK_VERSION=0.12.0 CHECK_TARBALL="check-${CHECK_VERSION}.tar.gz" CHECK_DIR="check-${CHECK_VERSION}" +echo "building and installing check" >&2 + ( cd "$TEMP" && wget "https://github.com/libcheck/check/releases/download/${CHECK_VERSION}/${CHECK_TARBALL}" && - tar xvfz "${CHECK_TARBALL}" && + tar xfz "${CHECK_TARBALL}" && cd "${CHECK_DIR}" && ./configure --prefix="$CHECK_PREFIX" && make && make install -) || die "check build failed" +) >$TEMP/cmake-build.log 2>&1 + +RESULT=$? +if [[ $RESULT -ne 0 ]]; then + cat >&2 <&2 +fi + +exit $RESULT diff --git a/ci/local-run.sh b/ci/local-run.sh new file mode 100755 index 000000000..ecc90c3ec --- /dev/null +++ b/ci/local-run.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +## This file is potentially useful for doing a clean environment build on MacOS ## +## This is my personal way of running the build, YMMV - jsimms ## + +set -euo pipefail +IFS=$'\n\t' + +die() { echo "fatal: $*" >&2; exit 1; } + +TOPLEVEL=$(git -C "$(cd "$(dirname "$0")" >/dev/null || exit 1; pwd)" rev-parse --show-toplevel) || die "TOPLEVEL fail" + +cd "$TOPLEVEL" + +TEMP="$(mktemp -d -t TEMP.XXXXXXX)" || die "failed to make tmpdir" +cleanup() { [[ -n "${TEMP:-}" ]] && rm -rf "${TEMP}"; } +trap cleanup EXIT + +BUILD_PATH=( + "$HOME/.cargo/bin" + "/usr/local/bin" + "/usr/local/sbin" + "/usr/bin" + "/usr/sbin" + "/bin" + "/sbin" + "/opt/X11/bin" + "/usr/X11R6/bin" +) + +PATH=$(echo "${BUILD_PATH[@]}"|tr ' ' ':') + +cat >&2 <. However, when interfacing with system calls and libraries - you cannot get away from using int and char. -- Use bool for boolean variables. You have to include -- If memory usage or alignment is a concern, avoid using a bool as type for - struct member names. Instead use unsigned 1-bit bit field. e.g. - struct foo { - unsigned is_bar:1; - }; - However, if neither memory usage or alignment will be significantly impacted - by the struct, opt for using bool for the sake of readability. -- Always use size_t type when dealing with sizes of objects or memory ranges. -- Your code should be 64-bit and 32-bit friendly. Bear in mind problems - of printing, comparisons, and structure alignment. You have to include - to get generic format specifier macros for printing. - -- 80 column line limit. -- If you have to wrap a long statement (> 80 column), put the operator at the - end of the line and indent the next line at the same column as the arguments - in the previous column. Eg: - while (cnt < 20 && this_variable_name_is_too_long && - ep != NULL) { - z = a + really + long + statement + that + needs + three + lines + - gets + indented + on + the + same + column + as + the + - previous + column - } - - and: - - int a = function(param_a, param_b, param_c, param_d, param_e, param_f, - param_g, param_h, param_i, param_j, param_k, param_l); - -- Always use braces for all conditional blocks (if, switch, for, while, do). - This holds good even for single statement conditional blocks. Eg: - if (cond) { - stmt; - } -- Placement of braces for non-function statement blocks - put opening brace - last on the line and closing brace first. Eg: - if (x is true) { - we do y - } -- Placement of brace for functions - put the opening brace at the beginning - of the next line and closing brace first. This is useful because several - tools look for opening brace in column one to find beginning of C - functions. Eg: -int -function(int x) -{ - body of the function -} - -- Closing brace is empty on a line of its own, except in cases where it is - followed by a continuation of the same statement, i.e. a "while" in a - do-statement or an "else" in an if-statement, like this: - do { - body of do-loop - } while (condition); - - and, - - if (x == y) { - .. - } else if (x > y) { - ... - } else { - .... - } - -- Column align switch keyword and the corresponding case/default keyword. Eg: - switch (alphabet) { - case 'a': - case 'b': - printf("I am a or b\n"); - break; - - default: - break; - } - -- Forever loops are done with for, and not while. Eg: - for (;;) { - stmt; - } - -- Don't use a space after a function name. -- Do not needlessly surround the return expression with parentheses. -- Use space after keywords. Exceptions are sizeof, typeof, alignof and - __attribute__, which look like functions. -- Do not add spaces around (inside) parenthesized expressions. - s = sizeof( sizeof(*p)) ); /* bad example */ - s = sizeof(sizeof(*p)); /* good example */ -- Casts should not be followed by space. Eg: - int q = *(int *)&p -- There is no need to type cast when assigning a void pointer to a non-void - pointer, or vice versa. -- Avoid using goto statements. However there are some exceptions to this rule - when a single goto label within a function and one or more goto statements - come in handy when a function exits from multiple locations and some common - work such as cleanup has to be done. Eg: -int -fun(void) -{ - int result = 0; - char *buffer; - - buffer = malloc(1024); - if (buffer == NULL) { - return -1; - } - - if (condition1) { - while (loop1) { - ... - } - result = 1; - goto out; - } - - ... -out: - free(buffer); - return result; -} -- When declaring pointer data, use '*' adjacent to the data name and not - adjacent to the type name. Eg: - int - function(int *p) - { - char *p; - - } -- Use one space around (on each side of) most binary and ternary operators, - such as any of these: - = + - < > * / % | & ^ <= >= == != ? : - but no space after unary operators: - & * + - ~ ! sizeof typeof alignof __attribute__ defined - no space before the postfix increment & decrement unary operators: - ++ -- - and no space around the '.' and "->" structure member operators. - -- 0 and NULL; use 0 for integers, 0.0 for doubles, NULL for pointers, and - '\0' for chars. -- Test pointers against NULL. E.g, use: - if (p == NULL) - - not: - - !(p) - -- Do not use ! for tests unless it is a boolean. E.g. use: - if (*p == '\0') - - not: - - if (!*p) - -- Don't use assignments inside if or while-conditions. E.g, use: - - struct foo *foo; - foo = malloc(sizeof(*foo)); - if (foo == NULL) { - return -1 - } - - not: - - struct foo *foo; - if ((foo = malloc(sizeof(*foo))) == NULL) { - return -1; - } - -- Don't ever use typedef for structure types. Typedefs are problematic - because they do not properly hide their underlying type; for example you - need to know if the typedef is the structure itself or a pointer to the - structure. In addition they must be declared exactly once, whereas an - incomplete structure type can be mentioned as many times as necessary. - Typedefs are difficult to use in stand-alone header files: the header - that defines the typedef must be included before the header that uses it, - or by the header that uses it (which causes namespace pollution), or - there must be a back-door mechanism for obtaining the typedef. -- The only exception for using a typedef is when you are defining a type - for a function pointer or a type for an enum. Eg: - - typedef void (*foo_handler_t)(int, void *); - - or: - - typedef enum types { - TYPE_1, - TYPE_2 - } types_t; - -- Use just one variable declaration per line when variables are part of a - struct. This leaves you room for a small comment on each item, explaining - its use. Declarations should also be aligned. Eg, use: - - struct foo { - int *foo_a; /* comment for foo_a */ - int foo_b; /* comment for foo_b */ - unsigned foo_c:1; /* comment for foo_c */ - }; - - and not: - - struct foo { - int *foo_a, foo_b; - unsigned foo_c:1; - }; - -- For variable declaration outside a struct, either collect all the - declarations of the same type on a single line, or use one variable - per line if the variables purpose needs to be commented. Eg: - char *a, *b, c; - - or: - - char *a, *b; - char c; /* comments for c */ - -- Avoid magic numbers because no-one has a clue (including the author) of - what it means after a month. - -- Function definitions should start the name of the function in column - one. This is useful because it makes searching for function definitions - fairly trivial. Eg: -static char * -concat(char *s1, char *s2) -{ - body of the function -} - -- Function and variables local to a file should be static. -- Separate two successive functions with one blank line. -- Include parameter names with their datypes in function declaration. Eg: -void function(int param); - -- Functions should be short and sweet, and do just one thing. They should - fit on one or two screenfuls of text (80 x 24 screen size), and do one - thing and do that well. - The maximum length of a function is inversely proportional to the - complexity and indentation level of that function. So, if you have a - conceptually simple function that is just one long (but simple) - case-statement, where you have to do lots of small things for a lot of - different cases, it's OK to have a longer function. - Another measure of the function is the number of local variables. They - shouldn't exceed 5-10, or you're doing something wrong. Re-think the - function, and split it into smaller pieces. A human brain can - generally easily keep track of about 7 different things, anything more - and it gets confused. You know you're brilliant, but maybe you'd like - to understand what you did 2 weeks from now. -- Use const for function parameters passed by reference, if the passed - pointer has no side effect. - -- C style comments only. Don't use // for single line comments. Instead - use /* ... */ style. -- For multi-line comments use the following style - /* - * This is the preferred style for multi-line - * comments in the Linux kernel source code. - * Please use it consistently. - * - * Description: A column of asterisks on the left side, - * with beginning and ending almost-blank lines. - */ - -- To comment out block of code spanning several lines use preprocessor - directive "#ifdef 0 ... #endif" - -- Please write a brief comment at the start of each source file, with the - file name and a line or two about the overall purpose of the file. - -- All major functions should have comments describing what they do at the - head of the function. Avoid putting comments in the function body unless - absolutely needed. If possible, add a comment on what sorts of arguments - the function gets, and what the possible values of arguments mean and - what they are used for and the significance of return value if there is - one. It is not necessary to duplicate in words the meaning of the C - argument declarations, if a C type is being used in its customary fashion. - If there is anything nonstandard about its use (such as an argument of - type char * which is really the address of the second character of a - string, not the first), or any possible values that would not work the - way one would expect (such as, that strings containing newlines are not - guaranteed to work), be sure to say so. Eg: - -/* - * Try to acquire a physical address lock while a pmap is locked. If we - * fail to trylock we unlock and lock the pmap directly and cache the - * locked pa in *locked. The caller should then restart their loop in case - * the virtual to physical mapping has changed. - * - * Returns 0 on success and -1 on failure. - */ -int -vm_page_pa_tryrelock(pmap_t pmap, vm_paddr_t pa, vm_paddr_t *locked) -{ - ... - -- The comment on a function is much clearer if you use the argument names - to speak about the argument values. The variable name itself should be - lower case, but write it in upper case when you are speaking about the - value rather than the variable itself. Thus, “the inode number NODE_NUM” - rather than “an inode”. - -- Every struct definition should have an accompanying comment that - describes what it is for and how it should be used. - -- Finally, while comments are absolutely important to keep the code readable, - remember that the best code is self-documenting. Giving sensible names to - types and variables is much better than using obscure names that you must - then explain through comments. - -- Recommend using UPPERCASE for macro names. However, sometimes using - lowercase for macro names makes sense when macros masquerade as well-known - function calls. Eg, it makes sense to write the wrapper for the - standard free() function in lowercase to keep the readability - consistent: - -#define my_free(_p) do { \ - free(_p); \ - (_p) = NULL; \ -} while (0) - -- Use enums when defining more than one related constants. All enumeration - values are in UPPERCASE. -- Avoid macros as much as possible and use inline functions, enums and const - variables wherever you can. -- For macros encapsulating compound statements, right justify the backslashes - and enclose it in do { ... } while (0) -- For parameterized macros, all the parameters used in the macro body must - be surrounded by parentheses. Eg: - #define ADD_1(_x) ((_x) + 1) - -- Use sizeof(varname) instead of sizeof(type) whenever possible. Eg: - char *p; - p = malloc(sizeof(*p)); /* good example */ - p = malloc(sizeof(char)); /* bad example */ - -- All variables should be declared at the beginning of a scope block {..}. - It is even preferred to declare all variables at the beginning of the - function so that all the local variable declarations is in one place and - we can see the comprehensive list in one glance. -- Global structs should be declared at the top of the file in which they - are used, or in separate header files if they are used in multiple - source files. -- Declarations of external functions and functions to appear later in the - source file should all go in one place near the beginning of the file, - somewhere before the first function definition in the file or else - should go in a header file. -- Use of extern should be considered as evil, if it is used in header files - to reference global variables. -- Don’t put extern declarations inside functions. - -- Usually every *.c file should have an associated *.h file. There are some - exceptions to this rule, such as unit tests and small *.c files containing - just the main() function. -- Every header file in the source code must have preprocessor conditional - to prevent the header file from being scanned multiple times and avoiding - mutual dependency cycles. Alternatively you can use #pragma once directive, - as it avoids name clashes and increases the compile speed. Eg, for a - header file named foo.h, the entire contents of the header file must be - between the guard macros as follows: - -#ifndef _FOO_H_ -#define _FOO_H_ -... -#endif /* _FOO_H_ */ - -Or, - -#pragma once -#ifndef _FOO_H_ -#define _FOO_H_ -... -#endif /* _FOO_H_ */ - -- Don't use #include when a forward declaration would suffice. -- Functions defined in header files should be static inline. - -- Don’t make the program ugly just to placate GCC when extra warnings options - such as ‘-Wconversion’ or ‘-Wundef’ are used. These options can help in - finding bugs, but they can also generate so many false alarms that that - it hurts readability to silence them with unnecessary casts, wrappers, and - other complications. - -- Conditional compilation: when supporting configuration options already - known when building your program we prefer using if (... ) over conditional - compilation, as in the former case the compiler is able to perform more - extensive checking of all possible code paths. Eg, use: - - if (HAS_FOO) - ... - else - ... - -instead of: - - #ifdef HAS_FOO - ... - #else - ... - #endif - - A modern compiler such as GCC will generate exactly the same code in both - cases and of course, the former method assumes that HAS_FOO is defined as - either 0 or 1. - -- Finally, rules are rules. Sometimes they are sensible and sometimes not - and regardless of your preference, we would like you to follow them. - A project is easier to follow if all project contributors follow the style - rules so that they can all read and understand everyone's code easily. But - remember, like all good rules, they are exceptions where it makes sense not - to be too rigid on the grounds of common sense and consistency! diff --git a/docs/coding_style.rst b/docs/coding_style.rst new file mode 100644 index 000000000..0a6a7bdfe --- /dev/null +++ b/docs/coding_style.rst @@ -0,0 +1,583 @@ +Introduction +============ + +The C Style Guide is a set of guidelines and conventions that encourage +good code. While some suggestions are more strict than others, you should +always practice good judgement. + +If following the guide causes unnecessary hoop-jumping or otherwise +less-readable code, *readability trumps the guide*. However, if the more +readable variant comes with perils or pitfalls, readability may be +sacrificed. + +Consistency is crucial. Without consistent application, there simply is no style +to speak of [#fn1]_. Stay in sync with the rest of the codebase; when you want +to change a rule or style, change it everywhere. + +Contents +-------- + +.. toctree:: + :maxdepth: 1 + + coding_style + + +C Standard +========== + +- Use ``-std=c11`` when compiling +- Avoid ``_Atomic``, ``_Generic`` and ``_Thread_local``, for now. We will + embrace ``C11`` fully when Twitter's official ``GCC`` is bumped to 4.9. + +Indentation +=========== + +- Do not use literal tabs. Expand tabs to **four** spaces instead. +- Use **four** spaces for every indentation level. +- Do not use more than **four** levels of indentation unless there's a good + reason. +- Make sure that your editor does not leave space at the end of each line. +- When a block of code resembles a table, such as in a macro-block or struct + definition, start each field on a ``4k+1`` column, so all fields are **four** + space aligned. + + .. code-block:: c + + /* name type description */ + #define FOO_METRIC(ACTION) \ + ACTION( foo_free, METRIC_GAUGE, "# free foo" )\ + ACTION( foo_borrow, METRIC_COUNTER, "# foos borrowed" )\ + ACTION( foo_return, METRIC_COUNTER, "# foos returned" )\ + /* name starts on column 13 + * type starts on column 29 + * description on column 45 + */ + + struct foo { + struct foo *next; + struct mumble amumble; + int bar; + }; + /* type starts on column 5, name on column 21 */ + + +Switch alignment +---------------- + +Align the ``switch`` keyword and the corresponding ``case`` and ``default`` +keywords to the same column. For example: + +.. code-block:: c + + switch (alphabet) { + case 'a': + case 'b': + printf("I am a or b\n"); + break; + default: + break; + } + + +Naming +====== + +- Use ``snake_case`` for the names of variables, functions, and files. +- Use your own judgement when you name variables and be as spartan as possible, + abbreviation is common in C. + For example, do not use a name like ``this_variable_is_a_temporary_counter``. + + +Types +===== + +- For variables of the following types: + + - ``int`` + - ``char`` + - ``short`` + - ``long`` + + Prefer the following types declared in the ```` header: + + - ``int8_t`` + - ``uint8_t`` + - ``int16_t`` + - ``uint16_t`` + - ``int32_t`` + - ``uint32_t`` + - ``int64_t`` + - ``uint64_t`` + + The latter types give us more predictable value range and memory layout on + different platforms. + +- Use the `bool` type for boolean data. You have to include the ```` + header. +- Always use the ``size_t`` type when you work with: + + - Sizes of objects + - Memory ranges + + +Line length +=========== + +- Limit each line to 80 columns or less. +- If you have to wrap a longer statement, put the operator at the end of the + line and use **eight** spaces to indent the next line. Indentation on the + next level is not affected. For example: + + .. code-block:: c + + while (cnt < 20 && this_variable_name_is_too_long && + ep != NULL) { + z = a + really + long + statement + that + needs + + two + lines + gets + indented + four + spaces + + on + the + second + and + subsequent + lines; + } + + and: + + .. code-block:: c + + int a = function(param_a, param_b, param_c, param_d, param_e, param_f, + param_g, param_h, param_i, param_j, param_k, param_l); + + +Braces +====== + +- Always use braces for all conditional blocks (``if``, ``switch``, ``for``, + ``while``, and ``do``), even for single statement conditional blocks (remember + the ``goto fail`` bug by Apple?). For example: + + .. code-block:: c + + if (cond) { + stmt; + } + +- For non-function statement blocks, put the opening brace at the end of the + first line and the closing brace in a new line. For example: + + .. code-block:: c + + if (x) { + foo(); + } + +- For functions, put the opening brace at the beginning of the second line + and the closing brace in a new line. For example: + + .. code-block:: c + + int + function(int x) + { + body of the function + } + +- Place the closing brace in its own line, except when it is part of the same + statement, such as a ``while`` in a ``do``-statement or an ``else`` in + an ``if``-statement. For example: + + .. code-block:: c + + do { + body of do-loop + } while (condition); + + and, + + if (x == y) { + .. + } else if (x > y) { + ... + } else { + .... + } + + +Infinite loops +============= + +Create infinite loops with ``for`` statements, not ``while`` statements. +For example: + +.. code-block:: c + + for (;;) { + stmt; + } + + +Spaces +====== + +- Do not use a space after a function name. +- Use space after keywords, except after the ``sizeof``, ``typeof``, ``alignof`` + and ``__attribute__`` keywords, since they are used like functions. +- Do not add spaces inside parenthesized expressions. For example: + + .. code-block:: c + + s = sizeof( sizeof(*p)) ); /* Bad example */ + s = sizeof(sizeof(*p)); /* Good example */ + +- When declaring pointers, place the asterisk ('*'') adjacent to the variable + name, not the type name. For example: + + .. code-block:: c + + int + function(int *p) + { + char *p; + body of the function + } + +- Use one space around most binary and ternary operators, such as any of these: + + ``=`` ``+`` ``-`` ``<`` ``>`` ``*`` ``/`` ``%`` ``|`` ``&`` ``^`` + ``<=`` ``>=`` ``==`` ``!=`` ``?`` ``:`` + + Do not add spaces after unary operators: + + ``&`` ``*`` ``+`` ``-`` ``~`` ``!`` ``sizeof`` ``typeof`` ``alignof`` + ``__attribute__`` ``defined`` + + Do not add spaces before the postfix increment and decrement unary operators: + + ``++`` ``--`` + + Do not add spaces around the ``.`` and ``->`` structure member operators. + +- Do not add spaces after casts. For example: + + .. code-block:: c + + int q = *(int *)&p + + +Type definitions +================ + +In general, do not use ``typedef`` for the purpose of hiding structures. +Typedefs used this way are problematic because they do not properly hide their +underlying type; for example, you need to know if the typedef is the structure +itself or a pointer to the structure. In addition, they must be declared exactly +once, whereas an incomplete structure type can be mentioned as many times as +necessary. Typedefs are difficult to use in stand-alone header files: the header +that defines the typedef must be included before the header that uses it, or by +the header that uses it (which causes namespace pollution), or there must be a +back-door mechanism for obtaining the typedef. + +That said, ``typedef`` can be helpful sometimes. For example, it is routinely +used to clarify the nature of an argument or return value, which can be of a +rather generic type such as ``void *``. It is also common to rename ``enum``. + +To make ``typedef`` names more informative and regular, we use the following +suffixes: + + - ``_e`` for ``enum`` + - ``_f`` for floating point numbers, regardless of size + - ``_i`` for signed integers, regardless of size + - ``_u`` for unsigned integers, regardless of size + - ``_fn`` for function pointers + - ``_p`` for other pointer type + - ``_st`` for ``struct`` + + +Forward Declaration +=================== + +Prefer using forward declaration over including another header for type- +declaration only. Forward declaration such as ``struct request;`` is feasible +if none of its members are directly accessed, such as when it's used in function +declaration. + + +Functions +========= + +- Declare functions that are local to a file as static. +- Place function types in their own line preceding the function. For example: + + .. code-block:: c + + static char * + function(int a1, int a2, float fl, int a4) + { + ... + +- Separate two successive functions with one blank line. +- Include parameter names with their dataypes in the function declaration. For + example: + + .. code-block:: c + + void function(int param); + +- When you use a wrapper function, name the wrapped function with the same name + as the wrapper function preceded by an underscore ('_'). Wrapped functions + are usually static. For example: + + .. code-block:: c + + static int + _fib(int n) + { + ... + } + + int + fib(int n) + { + ... + _fib(n); + ... + } + +- Create functions that are short and sweet. Functions should do just one + thing and fit on one or two screenfuls of text (80x24 screen size). + + The maximum length of a function is inversely proportional to the + complexity and indentation level of that function. So, if you have a + conceptually simple function that is just one long (but simple) + case-statement, where you have to do lots of small things for many + different cases, it is acceptable to have a longer function. + + Another measure of function complexity is the number of local variables. They + should not exceed 5-10. If your function has more than that, re-think the + function and split it into smaller pieces. A human brain can + generally easily keep track of about seven different things; anything more + and it gets confused. You may need to come back to your function and + understand what you did two weeks from now. + + +Goto statements +=============== + +- Use ``goto`` statements judiciously. Never use them to jump out of the + current function. Almost the only case where ``goto`` statements are helpful + is when a flow can exit from multiple locations within a function, and the + same clean-up logic applies to all of them. + + + .. code-block:: c + + int + fun(void) + { + int result = 0; + char *buffer; + buffer = malloc(1024); + if (buffer == NULL) { + return -1; + } + if (condition1) { + while (loop1) { + ... + } + result = 1; + goto out; + } + ... + out: + free(buffer); + return result; + } + + + +Comments +======== + +- Do not use ``//`` for single line comments. Instead, use the ``/* ... */`` + style. + +- For multi-line comments, use the following style: + + .. code-block:: c + + /* + * This is the preferred style for multi-line + * comments in the Linux kernel source code. + * Please use it consistently. + * + * Description: A column of asterisks on the left side, + * with beginning and ending almost-blank lines. + */ + +- To comment out blocks of code spanning several lines, use + ``#ifdef 0 ... #endif``. + +- Add comments before all major functions to describe what they do. Do not put + comments in the function body unless absolutely needed. For example: + + .. code-block:: c + + /* + * Try to acquire a physical address lock while a pmap is locked. If we + * fail to trylock we unlock and lock the pmap directly and cache the + * locked pa in *locked. The caller should then restart their loop in + * case the virtual to physical mapping has changed. + */ + int + vm_page_pa_tryrelock(pmap_t pmap, vm_paddr_t pa, vm_paddr_t *locked) + { + ... + + + +Other naming conventions +======================== + +- Use UPPERCASE for macro names. + +- Use ``enum`` to define several related constants. Use UPPERCASE for all + enumeration values. + +- Avoid macros as much as possible and use inline functions wherever you can. + +- For macros encapsulating compound statements, right-justify the backslashes + and enclose the statements in a ``do { ... } while (0)`` block. + +- For parameterized macros, add parentheses to all the parameters. For example: + + .. code-block:: c + + #define ADD_1(x) ((x) + 1) + + + +Inclusion +========= + +- Rule of thumb- local to global: first include header of the same name as + source, followed by headers in the same project, and external/system headers + last. +- Organize header inclusion in blocks, separated by blank line(s). For example, + headers that are shipped with the project and system headers should be in + separate clusters. +- Sort inclusions within the same block in alphabetic order. + + .. code-block:: c + + /* File: foo.c */ + #include "foo.h" /* first block: own header */ + + #include "bar.h" /* second block: headers from current project */ + #include "util/baz.h" + + #include /* third block: system/library headers */ + #include + #include + + +Structures +========== + +- To determine the size of a data structure, use some data of that type instead + of the type itself. For example: + + .. code-block:: c + + char *p; + p = malloc(sizeof(*p)); /* Good example */ + p = malloc(sizeof(char)); /* Bad example */ + +- Declare each variable in a structure in a separate line. Try to make the + structure readable by aligning the member names and comments using spaces. + Use a modest number of spaces if they suffice to align most of the member + names. Separate names that follow extremely long types with a single space. + + .. code-block:: c + + struct foo { + struct foo *next; /* List of active foo. */ + struct mumble amumble; /* Comment for mumble. */ + int bar; /* Try to align the comments. */ + struct verylongtypename *baz; /* Won't fit in 2 tabs. */ + }; + struct foo *foohead; /* Head of global foo list. */ + +- Declare major structures at the top of the file in which they are used, or + in separate header files if they are used in multiple source files. + Use of the structures should be by separate declarations and should be + ``extern`` if they are declared in a header file. + + +Pointers +======== + +- Use ``NULL`` as the null pointer constant (instead of ``0``). + +- Compare pointers to ``NULL``. For example: + + .. code-block:: c + + (p = f()) == NULL + + Do not compare to zero the integer. For example: + + .. code-block:: c + + !(p = f()) + +- Do not use ``!`` for comparisons (unless the variable is of boolean type). For + example: + + .. code-block:: c + + if (*p == '\0') + + The following snippet is a bad example: + + .. code-block:: c + + if (!*p) /* assume p is of type char * */ + +- Use ``const`` for function parameters if the pointer has no side effect. + +- Functions in charge of freeing an object should take a pointer to the intended + pointer to be freed, and set the pointer to the object to ``NULL`` before + returning. This prevents dangling pointers that are often discovered long + after ``free`` is called. + + .. code-block:: c + + void + destroy_buffer(struct buffer **pb) + { + free(*pb); + *pb = NULL; + } + +- Dynamically allocated structures should always initialize their members of + pointer type as soon as possible, to avoid the dangling pointer problem. + + +Macros +====== + +- Prefer ``static inline`` functions over macros. Macros often have unintended + side effects, for example: + + .. code-block:: c + + #define MAX(a,b) ((a) > (b) ? (a) : (b)) + + When used as in ``MAX(x++, y++)``, will increment either ``x`` or ``y`` twice, + which is probably not intended by the caller. + + +.. rubric:: Footnotes + +.. [#fn1] Frederick Brooks gave a definition of "style" in his book, The Design + of Design, which begins with "Style is a set of different repeated + microdecisions...". The book talked about the importance of Consistency in + the pages leading to this definition, starting from page 142, where the + author claimed that "consistency underlies all principles of quality". diff --git a/docs/modules/cc_log.rst b/docs/modules/cc_log.rst index 122dfe9c1..167fc4ff6 100644 --- a/docs/modules/cc_log.rst +++ b/docs/modules/cc_log.rst @@ -53,7 +53,7 @@ Synopsis void log_flush(struct logger *logger); - rstatus_i log_reopen(struct logger *logger); + rstatus_i log_reopen(struct logger *logger, char *target); Description ----------- @@ -95,18 +95,18 @@ Flush to file ^^^^^^^^^^^^^ .. code-block:: C - void log_flush(struct logger *logger); + size_t log_flush(struct logger *logger); -``log_flush`` writes as much data to the log file as possible, and updates the (read) marker in the ring buffer. Data that cannot be written to the file will be kept until next call. If the ring buffer or the file was never setup, no action is taken. +``log_flush`` writes as much data to the log file as possible, and updates the (read) marker in the ring buffer. Data that cannot be written to the file will be kept until next call. If the ring buffer or the file was never setup, no action is taken. Return the number of bytes flushed. Log reopen ^^^^^^^^^^ .. code-block:: C - rstatus_i log_reopen(struct logger *logger); + rstatus_i log_reopen(struct logger *logger, char *target); -``log_reopen`` reopens the log file according to ``name``, and does nothing if standard outputs are used. It returns ``CC_OK`` for success or ``CC_ERROR`` if reopen failed (at which point ``logger`` will no longer have a valid ``fd``). +``log_reopen`` reopens the log file according to ``name``, and does nothing if standard outputs are used. It returns ``CC_OK`` for success or ``CC_ERROR`` if reopen failed (at which point ``logger`` will no longer have a valid ``fd``). If ``target`` is specified function will rename original log file to the provided target filename and reopen the log file. This function can be used to reopen the log file when an exception has happened, or another party such as ``logrotate`` instructs the application to do so. Log rotation in a ``nocopytruncate`` manner- i.e. the content in the file is not copied, but the file is simply renamed- is more efficient in high-load systems. But doing so requires signaling the application to reopen the log file after renaming. This function makes it possible to achieve that when used with proper signal handling. diff --git a/docs/modules/cc_metric.rst b/docs/modules/cc_metric.rst index bfc6adad0..d220fd9db 100644 --- a/docs/modules/cc_metric.rst +++ b/docs/modules/cc_metric.rst @@ -111,10 +111,10 @@ Helper functions .. code-block:: C void metric_reset(struct metric sarr[], unsigned int nmetric); - size_t metric_print(char *buf, size_t nbuf, struct metric *m); + size_t metric_print(char *buf, size_t nbuf, char *fmt, struct metric *m); ``metric_reset`` resets the values of an array of metrics. -``metric_print`` prints the name and value of a metric, in human readable format, to buffer ``buf``, with a single space separating the two fields. This simple style is compatible with how Memcached currently reports metrics ([Memcached]_). Helper functions for other formats (e.g. Redis [Redis]_, StatsD [StatsD]_) may be introduced in the future. +``metric_print`` prints the name and value of a metric, in human readable format specified by ``fmt``, to buffer ``buf``. Update diff --git a/docs/modules/cc_option.rst b/docs/modules/cc_option.rst index ecde3dbe6..20ab0c0f0 100644 --- a/docs/modules/cc_option.rst +++ b/docs/modules/cc_option.rst @@ -40,6 +40,7 @@ Data Structure typedef enum option_type { OPTION_TYPE_BOOL, OPTION_TYPE_UINT, + OPTION_TYPE_FPN, OPTION_TYPE_STR, OPTION_TYPE_SENTINEL } option_type_e; @@ -47,6 +48,7 @@ Data Structure typedef union option_val { bool vbool; uintmax_t vuint; + double vfpn; char *vstr; } option_val_u; @@ -59,7 +61,7 @@ Data Structure char *description; }; -The core data structure ``struct option`` has six members. ``name`` and ``description`` help identify and explain the purpose of the option. ``type`` decides how input should be interpreted, which currently can be boolean, unsigned integer or C string. Both the default value and current value are kept around, with values matching the type. Keeping the default separately will make it easy to reset the option to original. Finally, boolean ``set`` tells if an option has been set, and thus usable. +The core data structure ``struct option`` has six members. ``name`` and ``description`` help identify and explain the purpose of the option. ``type`` decides how input should be interpreted, which currently can be boolean, unsigned integer, double or C string. Both the default value and current value are kept around, with values matching the type. Keeping the default separately will make it easy to reset the option to original. Finally, boolean ``set`` tells if an option has been set, and thus usable. Synopsis -------- @@ -71,8 +73,8 @@ Synopsis rstatus_i option_load_file(FILE *fp, struct option options[], unsigned int nopt); void option_print(struct option *opt); - void option_printall(struct option options[], unsigned int nopt); - void option_printall_default(struct option options[], unsigned int nopt); + void option_print_all(struct option options[], unsigned int nopt); + void option_describe_all(struct option options[], unsigned int nopt); void option_free(struct option options[], unsigned int nopt); @@ -118,8 +120,8 @@ Print option info .. code-block:: C void option_print(struct option *opt); - void option_printall(struct option options[], unsigned int nopt); - void option_printall_default(struct option options[], unsigned int nopt); + void option_print_all(struct option options[], unsigned int nopt); + void option_describe_all(struct option options[], unsigned int nopt); Examples diff --git a/include/buffer/cc_buf.h b/include/buffer/cc_buf.h index 4fe111dd3..ce7a6053b 100644 --- a/include/buffer/cc_buf.h +++ b/include/buffer/cc_buf.h @@ -38,25 +38,25 @@ extern "C" { /* name type default description */ -#define BUF_OPTION(ACTION) \ - ACTION( buf_init_size, OPTION_TYPE_UINT, BUF_DEFAULT_SIZE, "default size when buf is created" )\ - ACTION( buf_poolsize, OPTION_TYPE_UINT, BUF_POOLSIZE, "buf pool size" ) +#define BUF_OPTION(ACTION) \ + ACTION( buf_init_size, OPTION_TYPE_UINT, BUF_DEFAULT_SIZE, "init buf size incl header" )\ + ACTION( buf_poolsize, OPTION_TYPE_UINT, BUF_POOLSIZE, "buf pool size" ) typedef struct { BUF_OPTION(OPTION_DECLARE) } buf_options_st; /* name type description */ -#define BUF_METRIC(ACTION) \ - ACTION( buf_curr, METRIC_GAUGE, "# buf allocated" )\ - ACTION( buf_active, METRIC_GAUGE, "# buf in use/borrowed" )\ - ACTION( buf_create, METRIC_COUNTER, "# buf creates" )\ - ACTION( buf_create_ex, METRIC_COUNTER, "# buf create exceptions")\ - ACTION( buf_destroy, METRIC_COUNTER, "# buf destroys" )\ - ACTION( buf_borrow, METRIC_COUNTER, "# buf borrows" )\ - ACTION( buf_borrow_ex, METRIC_COUNTER, "# buf borrow exceptions")\ - ACTION( buf_return, METRIC_COUNTER, "# buf returns" )\ - ACTION( buf_memory, METRIC_GAUGE, "memory allocated to buf") +#define BUF_METRIC(ACTION) \ + ACTION( buf_curr, METRIC_GAUGE, "# buf allocated" )\ + ACTION( buf_active, METRIC_GAUGE, "# buf in use/borrowed" )\ + ACTION( buf_create, METRIC_COUNTER, "# buf creates" )\ + ACTION( buf_create_ex, METRIC_COUNTER, "# buf create exceptions" )\ + ACTION( buf_destroy, METRIC_COUNTER, "# buf destroys" )\ + ACTION( buf_borrow, METRIC_COUNTER, "# buf borrows" )\ + ACTION( buf_borrow_ex, METRIC_COUNTER, "# buf borrow exceptions" )\ + ACTION( buf_return, METRIC_COUNTER, "# buf returns" )\ + ACTION( buf_memory, METRIC_GAUGE, "memory alloc'd to buf including header" ) typedef struct { BUF_METRIC(METRIC_DECLARE) @@ -103,7 +103,7 @@ void buf_destroy(struct buf **buf); /* Size of data that has yet to be read */ static inline uint32_t -buf_rsize(struct buf *buf) +buf_rsize(const struct buf *buf) { ASSERT(buf->rpos <= buf->wpos); @@ -112,37 +112,38 @@ buf_rsize(struct buf *buf) /* Amount of room left in buffer for writing new data */ static inline uint32_t -buf_wsize(struct buf *buf) +buf_wsize(const struct buf *buf) { ASSERT(buf->wpos <= buf->end); return (uint32_t)(buf->end - buf->wpos); } -/* Total capacity of given buf */ +/* Total size of given buf, including header */ static inline uint32_t -buf_size(struct buf *buf) +buf_size(const struct buf *buf) { ASSERT(buf->begin < buf->end); - return (uint32_t)(buf->end - (char*)buf); + return (uint32_t)(buf->end - (char *)buf); } +/* Size of given buf, not including header */ static inline uint32_t -buf_capacity(struct buf *buf) +buf_capacity(const struct buf *buf) { ASSERT(buf->begin < buf->end); return (uint32_t)(buf->end - buf->begin); } -/* new capacity needed to append count bytes to the buffer */ +/* new capacity needed to write count bytes to the buffer */ static inline uint32_t -buf_new_cap(struct buf *buf, uint32_t count) +buf_new_cap(const struct buf *buf, uint32_t count) { ASSERT(buf->begin <= buf->wpos); - return buf->wpos - buf->begin + count; + return count <= buf_wsize(buf) ? 0 : count - buf_wsize(buf); } static inline void diff --git a/include/buffer/cc_dbuf.h b/include/buffer/cc_dbuf.h index 185de4734..07ad34f96 100644 --- a/include/buffer/cc_dbuf.h +++ b/include/buffer/cc_dbuf.h @@ -29,8 +29,8 @@ extern "C" { /* name type default description */ -#define DBUF_OPTION(ACTION) \ - ACTION( dbuf_max_power, OPTION_TYPE_UINT, DBUF_DEFAULT_MAX, "max number of doubling" ) +#define DBUF_OPTION(ACTION) \ + ACTION( dbuf_max_power, OPTION_TYPE_UINT, DBUF_DEFAULT_MAX, "max number of doubles") typedef struct { DBUF_OPTION(OPTION_DECLARE) @@ -43,7 +43,7 @@ typedef struct { ACTION( dbuf_double, METRIC_COUNTER, "# double completed" )\ ACTION( dbuf_double_ex, METRIC_COUNTER, "# double failed" )\ ACTION( dbuf_shrink, METRIC_COUNTER, "# shrink completed" )\ - ACTION( dbuf_shrink_ex, METRIC_COUNTER, "# srhink failed" )\ + ACTION( dbuf_shrink_ex, METRIC_COUNTER, "# shrink failed" )\ ACTION( dbuf_fit, METRIC_COUNTER, "# fit completed" )\ ACTION( dbuf_fit_ex, METRIC_COUNTER, "# fit failed" ) diff --git a/include/cc_bstring.h b/include/cc_bstring.h index b82c8b3f5..86c1aadb7 100644 --- a/include/cc_bstring.h +++ b/include/cc_bstring.h @@ -38,15 +38,14 @@ struct bstring { #define str2bstr(_str) (struct bstring){ sizeof(_str) - 1, (_str) } #define null_bstring (struct bstring){ 0, NULL } -#define bstring_set_text(_str, _text) do { \ - (_str)->len = (uint32_t)(sizeof(_text) - 1); \ - (_str)->data = (_text); \ +#define bstring_set_literal(_str, _literal) do { \ + (_str)->len = (uint32_t)(sizeof(_literal) - 1); \ + (_str)->data = (_literal); \ } while (0); -/* TODO(yao): rename this */ -#define bstring_set_raw(_str, _raw) do { \ - (_str)->len = (uint32_t)(cc_strlen(_raw)); \ - (_str)->data = (char *)(_raw); \ +#define bstring_set_cstr(_str, _cstr) do { \ + (_str)->len = (uint32_t)(cc_strlen(_cstr)); \ + (_str)->data = (char *)(_cstr); \ } while (0); void bstring_init(struct bstring *str); @@ -59,7 +58,6 @@ int bstring_compare(const struct bstring *s1, const struct bstring *s2); struct bstring *bstring_alloc(uint32_t size); void bstring_free(struct bstring **bstring); -/* TODO(yao): is this endian thing really useful? */ /* efficient implementation of string comparion of short strings */ #define str2cmp(m, c0, c1) \ (m[0] == c0 && m[1] == c1) @@ -67,70 +65,79 @@ void bstring_free(struct bstring **bstring); #define str3cmp(m, c0, c1, c2) \ (m[0] == c0 && m[1] == c1 && m[2] == c2) -#ifdef CC_LITTLE_ENDIAN - #define str4cmp(m, c0, c1, c2, c3) \ - (*(uint32_t *) m == ((c3 << 24) | (c2 << 16) | (c1 << 8) | c0)) + ((m[0] << 24 | m[1] << 16 | m[2] << 8 | m[3]) == \ + ((c0 << 24) | (c1 << 16) | (c2 << 8) | c3)) #define str5cmp(m, c0, c1, c2, c3, c4) \ (str4cmp(m, c0, c1, c2, c3) && (m[4] == c4)) #define str6cmp(m, c0, c1, c2, c3, c4, c5) \ - (str4cmp(m, c0, c1, c2, c3) && \ - (((uint32_t *) m)[1] & 0xffff) == ((c5 << 8) | c4)) + (str5cmp(m, c0, c1, c2, c3, c4) && m[5] == c5) #define str7cmp(m, c0, c1, c2, c3, c4, c5, c6) \ - (str6cmp(m, c0, c1, c2, c3, c4, c5) && (m[6] == c6)) + (str6cmp(m, c0, c1, c2, c3, c4, c5) && m[6] == c6) #define str8cmp(m, c0, c1, c2, c3, c4, c5, c6, c7) \ (str4cmp(m, c0, c1, c2, c3) && \ - (((uint32_t *) m)[1] == ((c7 << 24) | (c6 << 16) | (c5 << 8) | c4))) + (m[4] << 24 | m[5] << 16 | m[6] << 8 | m[7]) == \ + ((c4 << 24) | (c5 << 16) | (c6 << 8) | c7)) #define str9cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8) \ (str8cmp(m, c0, c1, c2, c3, c4, c5, c6, c7) && m[8] == c8) #define str10cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9) \ - (str8cmp(m, c0, c1, c2, c3, c4, c5, c6, c7) && \ - (((uint32_t *) m)[2] & 0xffff) == ((c9 << 8) | c8)) + (str9cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8) && m[9] == c9) #define str11cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10) \ - (str10cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9) && (m[10] == c10)) + (str10cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9) && m[10] == c10) #define str12cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11) \ (str8cmp(m, c0, c1, c2, c3, c4, c5, c6, c7) && \ - (((uint32_t *) m)[2] == ((c11 << 24) | (c10 << 16) | (c9 << 8) | c8))) - -#else // BIG ENDIAN + (m[8] << 24 | m[9] << 16 | m[10] << 8 | m[11]) == \ + ((c8 << 24) | (c9 << 16) | (c10 << 8) | c11)) + +/* below is a more efficient implementation for little-endian only, it takes + * about 50% the cycles compared to the generic implementation above in the + * extreme cases (e.g. string length being multiples of 4), however, our + * profiling showed that string comparison does not contribute meaningfully to + * overall processing cost, both events and hashes are far more notable, and + * therefore we can choose the generic implementation until profiling results + * indicate otherwise. + */ +/* #define str4cmp(m, c0, c1, c2, c3) \ - (str3cmp(m, c0, c1, c2) && (m3 == c3)) + (*(uint32_t *) m == ((c3 << 24) | (c2 << 16) | (c1 << 8) | c0)) #define str5cmp(m, c0, c1, c2, c3, c4) \ (str4cmp(m, c0, c1, c2, c3) && (m[4] == c4)) #define str6cmp(m, c0, c1, c2, c3, c4, c5) \ - (str5cmp(m, c0, c1, c2, c3, c4) && m[5] == c5) + (str4cmp(m, c0, c1, c2, c3) && \ + (((uint32_t *) m)[1] & 0xffff) == ((c5 << 8) | c4)) #define str7cmp(m, c0, c1, c2, c3, c4, c5, c6) \ - (str6cmp(m, c0, c1, c2, c3, c4, c5) && m[6] == c6) + (str6cmp(m, c0, c1, c2, c3, c4, c5) && (m[6] == c6)) #define str8cmp(m, c0, c1, c2, c3, c4, c5, c6, c7) \ - (str7cmp(m, c0, c1, c2, c3, c4, c5, c6) && m[7] == c7) + (str4cmp(m, c0, c1, c2, c3) && \ + (((uint32_t *) m)[1] == ((c7 << 24) | (c6 << 16) | (c5 << 8) | c4))) #define str9cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8) \ (str8cmp(m, c0, c1, c2, c3, c4, c5, c6, c7) && m[8] == c8) #define str10cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9) \ - (str9cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8) && m[9] == c9) + (str8cmp(m, c0, c1, c2, c3, c4, c5, c6, c7) && \ + (((uint32_t *) m)[2] & 0xffff) == ((c9 << 8) | c8)) #define str11cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10) \ - (str10cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9) && m[10] == c10) + (str10cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9) && (m[10] == c10)) #define str12cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11) \ - (str11cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10) && m[11] == c11) - -#endif // CC_LITTLE_ENDIAN - + (str8cmp(m, c0, c1, c2, c3, c4, c5, c6, c7) && \ + (((uint32_t *) m)[2] == ((c11 << 24) | (c10 << 16) | (c9 << 8) | c8))) +*/ /* * Wrapper around common routines for manipulating C character strings @@ -163,6 +170,7 @@ void bstring_free(struct bstring **bstring); /* bstring to uint conversion */ rstatus_i bstring_atou64(uint64_t *u64, struct bstring *str); +rstatus_i bstring_atoi64(int64_t *i64, struct bstring *str); #ifdef __cplusplus } diff --git a/include/cc_define.h b/include/cc_define.h index b521b133e..6a207036f 100644 --- a/include/cc_define.h +++ b/include/cc_define.h @@ -27,13 +27,6 @@ extern "C" { # define CC_HAVE_SIGNAME 1 #endif - -#ifdef HAVE_BIG_ENDIAN -# define CC_BIG_ENDIAN 1 -#else -# define CC_LITTLE_ENDIAN 1 -#endif - #ifdef HAVE_STATS # define CC_STATS 1 #endif @@ -54,10 +47,18 @@ extern "C" { # define CC_BACKTRACE 1 #endif +#ifdef HAVE_ACCEPT4 +# define CC_ACCEPT4 1 +#endif + #ifdef HAVE_DEBUG_MM #define CC_DEBUG_MM 1 #endif +#ifdef HAVE_ITT_INSTRUMENTATION +#define CC_ITT 1 +#endif + #define CC_OK 0 #define CC_ERROR -1 diff --git a/include/cc_event.h b/include/cc_event.h index 2f38caa6b..f5a31d281 100644 --- a/include/cc_event.h +++ b/include/cc_event.h @@ -26,8 +26,6 @@ extern "C" { #include -#define EVENT_SIZE 1024 - #define EVENT_READ 0x0000ff #define EVENT_WRITE 0x00ff00 #define EVENT_ERR 0xff0000 @@ -51,7 +49,7 @@ void event_setup(event_metrics_st *metrics); void event_teardown(void); /* event base */ -struct event_base *event_base_create(int size, event_cb_fn cb); +struct event_base *event_base_create(int nevent, event_cb_fn cb); void event_base_destroy(struct event_base **evb); /* event control */ diff --git a/include/cc_itt.h b/include/cc_itt.h new file mode 100644 index 000000000..583c72d93 --- /dev/null +++ b/include/cc_itt.h @@ -0,0 +1,69 @@ +#pragma once + +#include + +#ifdef CC_ITT +#include "ittnotify.h" +#endif + +#ifdef __cplusplus +extern "C" { +#endif +#ifdef CC_ITT + +#define ITT_DOMAIN_NAME "cc_itt" + +#define cc_declare_itt_function(_keyword, _name) \ + _keyword __itt_heap_function _name + +#define cc_create_itt_malloc(_name) \ + _name = __itt_heap_function_create(#_name, ITT_DOMAIN_NAME) + +#define cc_create_itt_free(_name) \ + _name = __itt_heap_function_create(#_name, ITT_DOMAIN_NAME) + +#define cc_create_itt_realloc(_name) \ + _name = __itt_heap_function_create(#_name, ITT_DOMAIN_NAME) + +#define cc_itt_alloc(_itt_heap_f, _p, _s) do { \ + __itt_heap_allocate_begin(_itt_heap_f, (size_t)(_s), 0); \ + __itt_heap_allocate_end(_itt_heap_f, (void *)&(_p), (size_t)(_s), 0); \ +} while (0) + +#define cc_itt_zalloc(_itt_heap_f, _p, _s) do { \ + __itt_heap_allocate_begin(_itt_heap_f, (size_t)(_s), 1); \ + __itt_heap_allocate_end(_itt_heap_f, (void *)&(_p), (size_t)(_s), 1); \ +} while (0) + +#define cc_itt_free(_itt_heap_f, _p) do { \ + __itt_heap_free_begin(_itt_heap_f, _p); \ + __itt_heap_free_end(_itt_heap_f, _p); \ +} while (0) + +#define cc_itt_realloc(_itt_heap_f, _p, _np, _s) do { \ + __itt_heap_reallocate_begin(_itt_heap_f, _p, (size_t)(_s), 0); \ + __itt_heap_reallocate_end(_itt_heap_f, _p, (void *)&(_np), (size_t)(_s), 0); \ +} while (0) + +#define cc_itt_heap_internal_access() \ + __itt_heap_internal_access_begin() + +#define cc_itt_heap_internal_access_end() \ + __itt_heap_internal_access_end() + +#else +#define cc_declare_itt_function(_keyword, _name) +#define cc_create_itt_malloc(_name) +#define cc_create_itt_free(_name) +#define cc_create_itt_realloc(_name) +#define cc_itt_alloc(_itt_heap_f, _p, _s) +#define cc_itt_zalloc(_itt_heap_f, _p, _s) +#define cc_itt_free(_itt_heap_f, _p) +#define cc_itt_realloc(_itt_heap_f, _p, _np, _s) +#define cc_itt_heap_internal_access_begin() +#define cc_itt_heap_internal_access_end() +#endif /* CC_ITT */ + +#ifdef __cplusplus +} +#endif diff --git a/include/cc_log.h b/include/cc_log.h index d61fcff91..f308842d8 100644 --- a/include/cc_log.h +++ b/include/cc_log.h @@ -61,6 +61,10 @@ typedef struct { void log_setup(log_metrics_st *metrics); void log_teardown(void); +/* these two are for testing purposes only */ +log_metrics_st *log_metrics_create(void); +void log_metrics_destroy(log_metrics_st **p); + /** * Create a logger. If filename is NULL, created logger writes to stderr. * buf_cap is the size of the buffer used for pauseless logging. specify diff --git a/include/cc_mm.h b/include/cc_mm.h index 2d982196e..a9effa23b 100644 --- a/include/cc_mm.h +++ b/include/cc_mm.h @@ -66,6 +66,9 @@ extern "C" { #define cc_munmap(_p, _s) \ _cc_munmap(_p, (size_t)(_s), __FILE__, __LINE__) +#define cc_alloc_usable_size(_p) \ + _cc_alloc_usable_size(_p, __FILE__, __LINE__) + void * _cc_alloc(size_t size, const char *name, int line); void * _cc_zalloc(size_t size, const char *name, int line); void * _cc_calloc(size_t nmemb, size_t size, const char *name, int line); @@ -74,6 +77,7 @@ void * _cc_realloc_move(void *ptr, size_t size, const char *name, int line); void _cc_free(void *ptr, const char *name, int line); void * _cc_mmap(size_t size, const char *name, int line); int _cc_munmap(void *p, size_t size, const char *name, int line); +size_t _cc_alloc_usable_size(void *ptr, const char *name, int line); #ifdef __cplusplus } diff --git a/include/cc_queue.h b/include/cc_queue.h index c8f8a5ce2..06bf9c31a 100644 --- a/include/cc_queue.h +++ b/include/cc_queue.h @@ -127,6 +127,7 @@ extern "C" { * _INSERT_TAIL - - + + + * _REMOVE_HEAD + - + - - * _REMOVE + + + + + + * _REINIT - - - + - * */ @@ -691,6 +692,23 @@ struct { \ (head2)->tqh_last = &(head2)->tqh_first; \ } while (0) +#define TAILQ_REINIT(head, var, field, offset) do { \ + TAILQ_FIRST((head)) = var; \ + TAILQ_FOREACH(var, head, field) { \ + if ((TAILQ_NEXT((var), field)) != NULL) { \ + TAILQ_NEXT((var), field) = \ + (void *)((char *)(TAILQ_NEXT((var), field)) + (offset));\ + } \ + if ((var) == TAILQ_FIRST(head)) { \ + (var)->field.tqe_prev = &TAILQ_FIRST(head); \ + } else { \ + (var)->field.tqe_prev = \ + (void *)((char *)((var)->field.tqe_prev) + (offset)); \ + } \ + (head)->tqh_last = &TAILQ_NEXT((var), field); \ + } \ +} while (0) + /* * Circular queue declarations. */ diff --git a/include/cc_stats_log.h b/include/cc_stats_log.h new file mode 100644 index 000000000..b33972770 --- /dev/null +++ b/include/cc_stats_log.h @@ -0,0 +1,38 @@ +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + + +#define STATS_LOG_FILE NULL /* default log file */ +#define STATS_LOG_NBUF 0 /* default log buf size */ + +/* name type default description */ +#define STATS_LOG_OPTION(ACTION) \ + ACTION( stats_log_file, OPTION_TYPE_STR, NULL, "file storing stats" )\ + ACTION( stats_log_nbuf, OPTION_TYPE_UINT, STATS_LOG_NBUF, "stats log buf size" ) + +typedef struct { + STATS_LOG_OPTION(OPTION_DECLARE) +} stats_log_options_st; + + +/* dump stats as CSV records into a log file, this allows metrics to be captured + * locally without setting up an observability infrastructure + */ +void stats_log_setup(stats_log_options_st *options); +void stats_log_teardown(void); + +void stats_log(struct metric metrics[], unsigned int nmetric); + +void stats_log_flush(void); + + +#ifdef __cplusplus +} +#endif + diff --git a/include/cc_util.h b/include/cc_util.h index d9c13955b..f8a770059 100644 --- a/include/cc_util.h +++ b/include/cc_util.h @@ -86,6 +86,9 @@ extern "C" { #define cc_strlen(_s) \ strlen((char *)(_s)) +#define cc_strnlen(_s, _n) \ + strnlen((char *)(_s), (size_t)(_n)) + #define cc_strcmp(_s1, _s2) \ strcmp((char *)(_s1), (char *)(_s2)) diff --git a/include/rust/cc_log_rs.h b/include/rust/cc_log_rs.h new file mode 100644 index 000000000..00eaf96f4 --- /dev/null +++ b/include/rust/cc_log_rs.h @@ -0,0 +1,75 @@ +/* ccommon - a cache common library. + * Copyright (C) 2013 Twitter, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +/* NOTE: for documentation see ccommon/rust/ccommon_rs/src/log.rs */ + +typedef enum log_level_rs { + LOG_LEVEL_ERROR = 1, + LOG_LEVEL_WARN, + LOG_LEVEL_INFO, + LOG_LEVEL_DEBUG, + LOG_LEVEL_TRACE, +} log_level_rs_e; + + +typedef enum log_status_rs { + /* Good work! */ + LOG_STATUS_OK = 0, + /* An action that requires log_rs_is_setup() to be true, but it isn't + * i.e. you need to call log_rs_setup() before whatever you just tried to do. */ + LOG_STATUS_NOT_SETUP_ERROR, + /* We could not register as the backend for the log crate . + * This state is unrecoverable. */ + LOG_STATUS_REGISTRATION_FAIL, + /* Returned when there is already a logger set up for rust. */ + LOG_STATUS_ALREADY_SET_ERROR, + /* Data was expected to be valid UTF8 but was not */ + LOG_STATUS_INVALID_UTF8, + /* Failed to create a logger instance */ + LOG_STATUS_CREATION_ERROR, + /* An unexpected error occurred, check stderr */ + LOG_STATUS_OTHER_FAILURE, + /* You suck at programming */ + LOG_STATUS_NULL_POINTER_ERROR, +} log_status_rs_e; + +struct log_config_rs { + uint32_t buf_size; + log_level_rs_e level; + struct bstring path; + struct bstring prefix; +}; + +struct log_handle_rs; + +struct log_handle_rs* log_create_handle_rs(struct log_config_rs *cfg); +log_status_rs_e log_shutdown_rs(struct log_handle_rs *handle, uint32_t timeout_ms); +void log_destroy_handle_rs(struct log_handle_rs **h); + +bool log_is_setup_rs(struct log_handle_rs *handle); + +#ifdef __cplusplus +} +#endif diff --git a/include/hash/cc_lookup3.h b/include/rust/cc_util_rs.h similarity index 77% rename from include/hash/cc_lookup3.h rename to include/rust/cc_util_rs.h index af6f8318c..bb354c129 100644 --- a/include/hash/cc_lookup3.h +++ b/include/rust/cc_util_rs.h @@ -1,6 +1,6 @@ /* * ccommon - a cache common library. - * Copyright (C) 2013 Twitter, Inc. + * Copyright (C) 2018 Twitter, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,12 +21,12 @@ extern "C" { #endif -#include +/* recursively remove all content under path then unlinks path. + * returns 0 on success, -1 on failure and sets errno. +*/ +int +cc_util_rm_rf_rs(const char *path); -#include -#include - -uint32_t hash_lookup3(const void *key, size_t length, const uint32_t initval); #ifdef __cplusplus } diff --git a/include/time/cc_timer.h b/include/time/cc_timer.h index 6a8c382ee..86b871a7a 100644 --- a/include/time/cc_timer.h +++ b/include/time/cc_timer.h @@ -49,6 +49,14 @@ struct duration; /* data structure to measure duration */ * relationship between this unit and nanosecond can be obtained via another * syscall */ + +enum duration_type { + DURATION_PRECISE, /* default */ + DURATION_FAST, + + MAX_DURATION_TYPE +}; + #ifdef OS_DARWIN struct duration { bool started; @@ -58,6 +66,7 @@ struct duration { }; #elif defined OS_LINUX struct duration { + enum duration_type type; bool started; bool stopped; struct timespec start; @@ -86,12 +95,12 @@ struct timeout { bool is_intvl; }; - /* update duration */ void duration_reset(struct duration *d); /* get a reading of duration and copy it without stopping the original timer */ void duration_snapshot(struct duration *s, const struct duration *d); void duration_start(struct duration *d); +void duration_start_type(struct duration *d, enum duration_type type); void duration_stop(struct duration *d); /* read duration */ double duration_ns(struct duration *d); @@ -99,6 +108,18 @@ double duration_us(struct duration *d); double duration_ms(struct duration *d); double duration_sec(struct duration *d); +static inline int +duration_compare(const void *lhs, const void *rhs) +{ + double lns = duration_ns((struct duration *)lhs); + double rns = duration_ns((struct duration *)rhs); + if (lns < rns) + return -1; + if (lns > rns) + return 1; + + return 0; +} /* * Not all possible granularity can be meaningfully used for sleep or event. diff --git a/rust/.gitignore b/rust/.gitignore new file mode 100644 index 000000000..331245bec --- /dev/null +++ b/rust/.gitignore @@ -0,0 +1,2 @@ +Cargo.lock +.cargo/ diff --git a/rust/CMakeLists.txt b/rust/CMakeLists.txt index 808fb4391..986ae5987 100644 --- a/rust/CMakeLists.txt +++ b/rust/CMakeLists.txt @@ -1,5 +1,12 @@ file(WRITE CMAKE_BINARY_DIR "${CMAKE_BINARY_DIR}\n") +execute_process( + COMMAND /usr/bin/env "CMAKE_BINARY_DIR=${CMAKE_BINARY_DIR}" "/bin/bash" "${CMAKE_CURRENT_LIST_DIR}/scripts/setup.sh" + WORKING_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}" + RESULT_VARIABLE CARGO_RESULT + OUTPUT_VARIABLE CARGO_CONFIG_OUT + ERROR_VARIABLE CARGO_CONFIG_OUT) + if(HAVE_RUST) add_subdirectory(ccommon_rs) endif() diff --git a/rust/Cargo.lock b/rust/Cargo.lock deleted file mode 100644 index 5ae870301..000000000 --- a/rust/Cargo.lock +++ /dev/null @@ -1,496 +0,0 @@ -[[package]] -name = "aho-corasick" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "ansi_term" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "atty" -version = "0.2.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", - "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "backtrace" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "backtrace-sys 0.1.23 (registry+https://github.com/rust-lang/crates.io-index)", - "cfg-if 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-demangle 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "backtrace-sys" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cc 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "bindgen" -version = "0.37.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cexpr 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "cfg-if 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "clang-sys 0.23.0 (registry+https://github.com/rust-lang/crates.io-index)", - "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)", - "env_logger 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", - "peeking_take_while 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro2 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "which 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "bitflags" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "cc" -version = "1.0.17" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "cc_binding" -version = "0.1.0" -dependencies = [ - "bindgen 0.37.4 (registry+https://github.com/rust-lang/crates.io-index)", - "failure 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "ccommon_rs" -version = "0.1.0" -dependencies = [ - "cc_binding 0.1.0", - "failure 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "cexpr" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "nom 3.2.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "cfg-if" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "clang-sys" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", - "libloading 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "clap" -version = "2.32.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", - "atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", - "bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", - "textwrap 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "env_logger" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", - "humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "failure" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "backtrace 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "failure_derive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "failure_derive" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", - "synstructure 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "glob" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "humantime" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "lazy_static" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "libc" -version = "0.2.42" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "libloading" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cc 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "log" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cfg-if 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "memchr" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "memchr" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "nom" -version = "3.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "proc-macro2" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "quick-error" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "quote" -version = "0.3.15" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "quote" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "proc-macro2 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "redox_syscall" -version = "0.1.40" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "redox_termios" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "regex" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "aho-corasick 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", - "thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "regex-syntax" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "rustc-demangle" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "strsim" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "syn" -version = "0.11.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", - "synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "synom" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "synstructure" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "termcolor" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "wincolor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "termion" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "textwrap" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "thread_local" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "ucd-util" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "unicode-width" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "unicode-xid" -version = "0.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "unicode-xid" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "unreachable" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "utf8-ranges" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "vec_map" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "void" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "which" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "winapi" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "wincolor" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[metadata] -"checksum aho-corasick 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f0ba20154ea1f47ce2793322f049c5646cc6d0fa9759d5f333f286e507bf8080" -"checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" -"checksum atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "2fc4a1aa4c24c0718a250f0681885c1af91419d242f29eb8f2ab28502d80dbd1" -"checksum backtrace 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "89a47830402e9981c5c41223151efcced65a0510c13097c769cede7efb34782a" -"checksum backtrace-sys 0.1.23 (registry+https://github.com/rust-lang/crates.io-index)" = "bff67d0c06556c0b8e6b5f090f0eac52d950d9dfd1d35ba04e4ca3543eaf6a7e" -"checksum bindgen 0.37.4 (registry+https://github.com/rust-lang/crates.io-index)" = "1b25ab82877ea8fe6ce1ce1f8ac54361f0218bad900af9eb11803994bf67c221" -"checksum bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d0c54bb8f454c567f21197eefcdbf5679d0bd99f2ddbe52e84c77061952e6789" -"checksum cc 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)" = "49ec142f5768efb5b7622aebc3fdbdbb8950a4b9ba996393cb76ef7466e8747d" -"checksum cexpr 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "42aac45e9567d97474a834efdee3081b3c942b2205be932092f53354ce503d6c" -"checksum cfg-if 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "efe5c877e17a9c717a0bf3613b2709f723202c4e4675cc8f12926ded29bcb17e" -"checksum clang-sys 0.23.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d7f7c04e52c35222fffcc3a115b5daf5f7e2bfb71c13c4e2321afe1fc71859c2" -"checksum clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b957d88f4b6a63b9d70d5f454ac8011819c6efa7727858f458ab71c756ce2d3e" -"checksum env_logger 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)" = "0e6e40ebb0e66918a37b38c7acab4e10d299e0463fe2af5d29b9cc86710cfd2a" -"checksum failure 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "934799b6c1de475a012a02dab0ace1ace43789ee4b99bcfbf1a2e3e8ced5de82" -"checksum failure_derive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c7cdda555bb90c9bb67a3b670a0f42de8e73f5981524123ad8578aafec8ddb8b" -"checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb" -"checksum humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0484fda3e7007f2a4a0d9c3a703ca38c71c54c55602ce4660c419fd32e188c9e" -"checksum lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e6412c5e2ad9584b0b8e979393122026cdd6d2a80b933f890dcd694ddbe73739" -"checksum libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" = "b685088df2b950fccadf07a7187c8ef846a959c142338a48f9dc0b94517eb5f1" -"checksum libloading 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9c3ad660d7cb8c5822cd83d10897b0f1f1526792737a179e73896152f85b88c2" -"checksum log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "61bd98ae7f7b754bc53dca7d44b604f733c6bba044ea6f41bc8d89272d8161d2" -"checksum memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "148fab2e51b4f1cfc66da2a7c32981d1d3c083a803978268bb11fe4b86925e7a" -"checksum memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "796fba70e76612589ed2ce7f45282f5af869e0fdd7cc6199fa1aa1f1d591ba9d" -"checksum nom 3.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05aec50c70fd288702bcd93284a8444607f3292dbdf2a30de5ea5dcdbe72287b" -"checksum peeking_take_while 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" -"checksum proc-macro2 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "77997c53ae6edd6d187fec07ec41b207063b5ee6f33680e9fa86d405cdd313d4" -"checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0" -"checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" -"checksum quote 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9949cfe66888ffe1d53e6ec9d9f3b70714083854be20fd5e271b232a017401e8" -"checksum redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "c214e91d3ecf43e9a4e41e578973adeb14b474f2bee858742d127af75a0112b1" -"checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" -"checksum regex 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "13c93d55961981ba9226a213b385216f83ab43bd6ac53ab16b2eeb47e337cf4e" -"checksum regex-syntax 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05b06a75f5217880fc5e905952a42750bf44787e56a6c6d6852ed0992f5e1d54" -"checksum rustc-demangle 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "76d7ba1feafada44f2d38eed812bd2489a03c0f5abb975799251518b68848649" -"checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550" -"checksum syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad" -"checksum synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6" -"checksum synstructure 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3a761d12e6d8dcb4dcf952a7a89b475e3a9d69e4a69307e01a470977642914bd" -"checksum termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "adc4587ead41bf016f11af03e55a624c06568b5a19db4e90fde573d805074f83" -"checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" -"checksum textwrap 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "307686869c93e71f94da64286f9a9524c0f308a9e1c87a583de8e9c9039ad3f6" -"checksum thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "279ef31c19ededf577bfd12dfae728040a21f635b06a24cd670ff510edd38963" -"checksum ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd2be2d6639d0f8fe6cdda291ad456e23629558d466e2789d2c3e9892bda285d" -"checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" -"checksum unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" -"checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" -"checksum unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" -"checksum utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "662fab6525a98beff2921d7f61a39e7d59e0b425ebc7d0d9e66d316e55124122" -"checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" -"checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" -"checksum which 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e84a603e7e0b1ce1aa1ee2b109c7be00155ce52df5081590d1ffb93f4f515cb2" -"checksum winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "773ef9dcc5f24b7d850d0ff101e542ff24c3b090a9768e03ff889fdef41f00fd" -"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" -"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -"checksum wincolor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "eeb06499a3a4d44302791052df005d5232b927ed1a9658146d842165c4de7767" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index ae5821dad..ccde17a74 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -8,8 +8,8 @@ members = [ [profile.release] opt-level = 3 debug = true -rpath = false -lto = true +# rpath = false +# lto = true debug-assertions = false codegen-units = 1 diff --git a/rust/cc_binding/Cargo.toml b/rust/cc_binding/Cargo.toml index 0ad06b1df..4182cbc6d 100644 --- a/rust/cc_binding/Cargo.toml +++ b/rust/cc_binding/Cargo.toml @@ -9,4 +9,3 @@ failure = "~0.1.1" [lib] name = "cc_binding" -crate-type = ["rlib", "dylib", "staticlib", "lib"] diff --git a/rust/cc_binding/build.rs b/rust/cc_binding/build.rs index 4e6dc243d..f3eda9c93 100644 --- a/rust/cc_binding/build.rs +++ b/rust/cc_binding/build.rs @@ -48,7 +48,10 @@ fn get_cmake_cache_value(binary_dir: &Path, key: &str) -> Result> } fn main() { - println!("cargo:rustc-link-lib=static=ccommon-1.2.0"); + println!("cargo:rustc-link-lib=static=ccommon-2.1.0"); + if cfg!(target_os = "macos") { + println!("cargo:rustc-link-lib=framework=Security"); + } let include_path = fs::canonicalize("./../../include").unwrap(); @@ -76,6 +79,10 @@ fn main() { println!("cargo:rustc-link-search=native={}", lib_dir); + if cfg!(target_os = "macos") { + println!("cargo:rustc-link-search=framework=/System/Library/Frameworks"); + } + let bindings = bindgen::Builder::default() .clang_args(vec![ "-I", include_path.to_str().unwrap(), diff --git a/rust/cc_binding/src/lib.rs b/rust/cc_binding/src/lib.rs index 2cf071191..458966182 100644 --- a/rust/cc_binding/src/lib.rs +++ b/rust/cc_binding/src/lib.rs @@ -1,3 +1,26 @@ +// ccommon - a cache common library. +// Copyright (C) 2018 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! This module contains the bindgen created classes. +//! PRO-TIP: If you want to look at the generated code, you can find it with: +//! +//! ```ignore +//! $ find . -name bindgen.rs +//! ``` +//! + #![allow(unknown_lints)] #![allow(clippy)] #![allow(clippy_pedantic)] @@ -6,3 +29,4 @@ #![allow(non_snake_case)] #![allow(dead_code)] include!(concat!(env!("OUT_DIR"), "/bindings.rs")); + diff --git a/rust/cc_binding/wrapper.h b/rust/cc_binding/wrapper.h index c5b21b148..02962be00 100644 --- a/rust/cc_binding/wrapper.h +++ b/rust/cc_binding/wrapper.h @@ -1,2 +1,20 @@ +/* ccommon - a cache common library. + * Copyright (C) 2018 Twitter, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + #include #include +#include +#include diff --git a/rust/ccommon_rs/CMakeLists.txt b/rust/ccommon_rs/CMakeLists.txt index f373e0938..8d0335b2e 100644 --- a/rust/ccommon_rs/CMakeLists.txt +++ b/rust/ccommon_rs/CMakeLists.txt @@ -1,4 +1,3 @@ file(WRITE CMAKE_BINARY_DIR "${CMAKE_BINARY_DIR}\n") cargo_build(NAME ccommon_rs) -add_dependencies(ccommon_rs_static ccommon-static) -add_dependencies(ccommon_rs_shared ccommon-shared) +add_dependencies(ccommon_rs ccommon-static) diff --git a/rust/ccommon_rs/Cargo.toml b/rust/ccommon_rs/Cargo.toml index 382bf2286..2b25168a0 100644 --- a/rust/ccommon_rs/Cargo.toml +++ b/rust/ccommon_rs/Cargo.toml @@ -3,10 +3,22 @@ name = "ccommon_rs" version = "0.1.0" authors = ["Jonathan Simms "] +[lib] +name = "ccommon_rs" +crate-type = ["staticlib", "rlib"] + [dependencies] cc_binding = { path = "../cc_binding" } - +chrono = "~0.4.4" +crossbeam = "~0.3.2" +failure = "~0.1" +failure_derive = "~0.1" +lazy_static = "~1.0" +libc = "~0.2" log = "~0.4" -libc = "~0.2.42" -failure = "~0.1.1" +rusty-fork = "~0.2.0" +tempfile = "~3.0" +thread-id = "~3.3" +thread_local = "~0.3.5" +time = "~0.1" diff --git a/rust/ccommon_rs/src/bstring.rs b/rust/ccommon_rs/src/bstring.rs index f8588d608..5a33f8078 100644 --- a/rust/ccommon_rs/src/bstring.rs +++ b/rust/ccommon_rs/src/bstring.rs @@ -91,8 +91,21 @@ impl BStr { pub fn as_ptr(&self) -> *mut CCbstring { self as *const _ as *mut _ } + + pub fn from_ref<'a>(ccb: &'a CCbstring) -> &'a Self { + unsafe { Self::from_ptr(ccb as *const CCbstring as *mut _) } + } + + pub fn to_utf8_str<'a>(&'a self) -> super::Result<&'a str> { + str::from_utf8(&self[..]).map_err(|e| e.into()) + } + + pub fn to_utf8_string(&self) -> super::Result { + self.to_utf8_str().map(|x| x.to_owned()) + } } + impl Deref for BStr { type Target = [u8]; @@ -263,6 +276,7 @@ impl BString { /// # Panics /// /// This method will panic if `src.len() != self.len()` + #[allow(dead_code)] #[inline] #[allow(dead_code)] fn copy_from_slice(&mut self, src: &[u8]) { @@ -271,12 +285,12 @@ impl BString { } #[inline] - fn as_bytes(&self) -> &[u8] { + pub fn as_bytes(&self) -> &[u8] { unsafe { raw_ptr_to_bytes(self.0) } } #[inline] - fn as_bytes_mut(&mut self) -> &mut [u8] { + pub fn as_bytes_mut(&mut self) -> &mut [u8] { unsafe { raw_ptr_to_bytes_mut(self.0) } } @@ -284,6 +298,14 @@ impl BString { fn len(&self) -> usize { unsafe { (*self.0).len as usize } } + + pub fn to_utf8_str<'a>(&'a self) -> super::Result<&'a str> { + str::from_utf8(self.as_bytes()).map_err(|e| e.into()) + } + + pub fn to_utf8_string(&self) -> super::Result { + self.to_utf8_str().map(|x| x.to_owned()) + } } impl Debug for BString { diff --git a/rust/ccommon_rs/src/lib.rs b/rust/ccommon_rs/src/lib.rs index 6d035da4d..639b47540 100644 --- a/rust/ccommon_rs/src/lib.rs +++ b/rust/ccommon_rs/src/lib.rs @@ -1,3 +1,44 @@ +// ccommon - a cache common library. +// Copyright (C) 2018 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + extern crate cc_binding; +extern crate chrono; +extern crate crossbeam; +#[macro_use] extern crate failure; +#[macro_use] +extern crate failure_derive; +extern crate lazy_static; +#[macro_use] +extern crate log as rslog; +extern crate tempfile; +extern crate time; +extern crate thread_local; +extern crate thread_id; + +#[cfg(test)] +#[macro_use] +extern crate rusty_fork; + +use std::result; + pub mod bstring; +pub mod log; +pub mod util; + +// like how guava provides enhancements for Int as "Ints" +pub mod ptrs; + +pub type Result = result::Result; diff --git a/rust/ccommon_rs/src/log/mod.rs b/rust/ccommon_rs/src/log/mod.rs new file mode 100644 index 000000000..224e1f469 --- /dev/null +++ b/rust/ccommon_rs/src/log/mod.rs @@ -0,0 +1,856 @@ +// ccommon - a cache common library. +// Copyright (C) 2018 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Threadsafe glue between the `log` crate and `cc_log`. +//! +//! The C side configures this module with a directory and base filename. +//! When a Rust thread calls one of the logging macros, a new logger is +//! created with a unique filename (based either on the thread's name or +//! its posix unique id) and stored in a thread local variable. At shutdown, +//! the struct that refers to the thread-local loggers is atomically +//! swapped out for a no-op logger, and the thread-local loggers are flushed +//! and shut down cleanly. +//! +//! This configuration is a shared-nothing lockless design...for _SPEED_. +//! +//! # Example +//! +//! Basic setup in an app that makes use of ccommon: +//! +//! ```ignore +//! #include +//! #include +//! #include +//! +//! #include +//! +//! #define PATH "/var/log/appname" +//! +//! static struct log_handle_rs *log_handle; +//! static struct log_config_rs log_config; +//! +//! +//! void +//! log_setup() +//! { +//! log_config.buf_size = 1024; +//! bstring_set_cstr(&log_config.prefix, "templog"); +//! bstring_set_cstr(&log_config.path, PATH); +//! log_config.level = LOG_LEVEL_TRACE; +//! +//! log_handle = log_create_handle_rs(&log_config); +//! ASSERT(log_handle != NULL); +//! ASSERT(log_is_setup_rs(log_handle)); +//! } +//! +//! void +//! log_teardown() +//! { +//! if (log_shutdown_rs(log_handle) != LOG_STATUS_OK) { +//! /* emit a warning about this */ +//! } +//! +//! log_destroy_handle_rs(&log_handle); +//! } +//! +//! ``` + +#![allow(dead_code)] + +pub use rslog::{Level, Log, SetLoggerError}; +use rslog::{Metadata, Record}; +pub use super::Result; +use cc_binding as bind; +use crossbeam::sync::ArcCell; +use failure; +use ptrs; +use rslog; +use bstring::BStr; +use std::cell::RefCell; +use std::ffi::CString; +use std::io::{Cursor, Write}; +use std::path::PathBuf; +use std::ptr; +use std::sync::Arc; +use std::thread; +use thread_id; +use thread_local::CachedThreadLocal; +use time; + + +// TODO(simms): add C-side setup code here. + +const PER_THREAD_BUF_SIZE: usize = 4096; + +#[derive(Fail, Debug)] +pub enum LoggingError { + #[fail(display = "logging already set up")] + LoggingAlreadySetUp, + + #[fail(display = "Other logger has already been set up with log crate")] + LoggerRegistrationFailure, + + #[fail( + display = "cc_log_create failed. see stderr for message. path: {}, buf_size: {}", + path, buf_size + )] + CreationError { path: String, buf_size: u32 }, + +} + +impl From for LoggingError { + fn from(_: SetLoggerError) -> Self { + LoggingError::LoggerRegistrationFailure + } +} + + +#[doc(hidden)] +pub struct CLogger(*mut bind::logger); + +impl CLogger { + pub unsafe fn from_raw(p: *mut bind::logger) -> super::Result { + ptrs::null_check(p).map(CLogger).map_err(|e| e.into()) + } + + pub unsafe fn write(&self, msg: &[u8]) -> bool { + let b = bind::log_write(self.0, msg.as_ptr() as *mut i8, msg.len() as u32); + if !b { + eprintln!("failed to write to log: {:#?}", &msg); + } + b + } + + pub unsafe fn flush(&self) { bind::log_flush(self.0); } + + pub unsafe fn open(path: &str, buf_size: u32) -> super::Result { + let p = bind::log_create(CString::new(path)?.into_raw(), buf_size); + + ptrs::lift_to_option(p) + .ok_or_else(|| LoggingError::CreationError {path: path.to_owned(), buf_size}.into()) + .map(CLogger) + } + + pub fn as_mut_ptr(&mut self) -> *mut bind::logger { self.0 } +} + +impl Drop for CLogger { + fn drop(&mut self) { + unsafe { bind::log_destroy(&mut self.0) } + } +} + +fn format(record: &Record, buf: &mut Vec) -> Result { + let tm = time::now_utc(); + + let mut curs = Cursor::new(buf); + + let ts = time::strftime("%Y-%m-%d %H:%M:%S", &tm).unwrap(); + + writeln!( + curs, + "{}.{:06} {:<5} [{}] {}", + ts, + tm.tm_nsec, + record.level().to_string(), + record.module_path().unwrap_or_default(), + record.args() + )?; + + Ok(curs.position() as usize) +} + +#[repr(u32)] +#[derive(Debug, PartialEq, PartialOrd, Eq)] +pub enum LoggerStatus { + OK = 0, + LoggerNotSetupError = 1, + RegistrationFailure = 2, + LoggerAlreadySetError = 3, + InvalidUTF8 = 4, + CreationError = 5, + OtherFailure = 6, + NullPointerError = 7, +} + +impl From for LoggerStatus { + fn from(e: LoggingError) -> Self { + match e { + LoggingError::LoggerRegistrationFailure => LoggerStatus::RegistrationFailure, + LoggingError::LoggingAlreadySetUp => LoggerStatus::LoggerAlreadySetError, + LoggingError::CreationError{..} => LoggerStatus::CreationError, + } + } +} + + +#[repr(usize)] +#[doc(hidden)] +#[derive(Debug, Eq, PartialEq)] +enum ModuleState { + UNINITIALIZED = 0, + INITIALIZING, + INITIALIZED, + FAILED, +} + +impl From for ModuleState { + fn from(u: usize) -> Self { + match u { + 0 => ModuleState::UNINITIALIZED, + 1 => ModuleState::INITIALIZING, + 2 => ModuleState::INITIALIZED, + 3 => ModuleState::FAILED, + _ => unreachable!() + } + } +} + +#[cfg(test)] +pub(in log) struct LogMetrics(*mut bind::log_metrics_st); + +#[cfg(test)] +impl LogMetrics { + pub fn new() -> Self { + let ptr = unsafe { bind::log_metrics_create() }; + assert!(!ptr.is_null()); + LogMetrics(ptr) + } + + pub fn as_mut_ptr(&mut self) -> *mut bind::log_metrics_st { self.0 } +} + +#[cfg(test)] +impl Drop for LogMetrics { + fn drop(&mut self) { + unsafe { bind::log_metrics_destroy(&mut self.0) } + } +} + +const DEFAULT_LOG_BASENAME: &str = "ccommon"; + +#[repr(C)] +pub struct LogConfig { + /// Path to the directory where we will write log files + path: String, + + /// The basis for log filenames. If `foobar` is given, + /// logs will be named `foobar.${thread_id}.log`. There will be one + /// log created per thread. If the thread is named, that will be used + /// as `thread_id` otherwise a unique identifier will be chosen. + prefix: String, + + /// What size buffer should the cc_log side use? + buf_size: u32, + + level: Level, +} + +#[derive(Clone, Debug)] +pub struct LogConfigBuilder { + path: Option, + prefix: Option, + buf_size: Option, + level: Option, +} + +impl Default for LogConfigBuilder { + fn default() -> Self { + LogConfigBuilder{ + path: None, + prefix: Some(String::from("ccommon")), + buf_size: Some(0), + level: Some(Level::Trace) + } + } +} + + +impl LogConfigBuilder { + pub fn path(&mut self, path: String) -> &mut Self { + let new = self; + new.path = Some(path); + new + } + + pub fn prefix(&mut self, prefix: String) -> &mut Self { + let new = self; + new.prefix = Some(prefix); + new + } + + pub fn buf_size(&mut self, buf: u32) -> &mut Self { + let new = self; + new.buf_size = Some(buf); + new + } + + pub fn level(&mut self, lvl: Level) -> &mut Self { + let new = self; + new.level = Some(lvl); + new + } + + pub fn build(&self) -> Result { + if self.path.is_none() { + bail!("path field must be set: {:#?}", self) + } + Ok(LogConfig{ + path: Clone::clone(&self.path).unwrap().to_owned(), + prefix: Clone::clone(&self.prefix).unwrap().to_owned(), + buf_size: Clone::clone(&self.buf_size).unwrap(), + level: Clone::clone(&self.level).unwrap(), + }) + } +} + +fn level_from_usize(u: usize) -> Option { + match u { + 1 => Some(Level::Error), + 2 => Some(Level::Warn), + 3 => Some(Level::Info), + 4 => Some(Level::Debug), + 5 => Some(Level::Trace), + _ => None, + } +} + +impl LogConfig { + #[doc(hidden)] + pub unsafe fn from_raw(ptr: *mut bind::log_config_rs) -> Result { + ptrs::lift_to_option(ptr) + .ok_or_else(|| ptrs::NullPointerError.into()) + .and_then(|ptr| { + let raw = *ptr; + + let path = BStr::from_ref(&raw.path).to_utf8_string()?; + let prefix = BStr::from_ref(&raw.prefix).to_utf8_string()?; + let buf_size = raw.buf_size; + let level = + match level_from_usize(raw.level as usize) { + Some(n) => n, + None => Level::Trace, + }; + + LogConfigBuilder::default() + .path(path) + .prefix(prefix) + .buf_size(buf_size) + .level(level) + .build() + }) + } + + fn to_path_buf(&self, thread_id: &str) -> PathBuf { + let mut pb = PathBuf::new(); + pb.push(&self.path); + pb.push(format!("{}.{}.log", self.prefix, thread_id)); + pb + } +} + + +struct PerThreadLog { + /// The underlying cc_log logger instance + clogger: CLogger, + /// The cached thread name or unique identifier + thread_name: String, + /// This buffer is used for preparing the message to be logged + buf: RefCell>, +} + +impl PerThreadLog { + fn for_current(cfg: &LogConfig) -> super::Result { + let tc = thread::current(); + let thread_name = + tc.name() + .map(|s| s.to_owned()) + .unwrap_or_else(|| { format!("{}", thread_id::get()) }); + + let clogger = unsafe { + CLogger::open(cfg.to_path_buf(&thread_name[..]).to_str().unwrap(), cfg.buf_size)? + }; + + let buf = RefCell::new(Vec::with_capacity(PER_THREAD_BUF_SIZE)); + + Ok(PerThreadLog{thread_name, clogger, buf}) + } +} + +unsafe impl Sync for PerThreadLog {} +unsafe impl Send for PerThreadLog {} + + +impl Log for PerThreadLog { + fn enabled(&self, _: &Metadata) -> bool { + true + } + + fn log(&self, record: &Record) { + if self.enabled(record.metadata()) { + let mut buf = self.buf.borrow_mut(); + let sz = format(record, &mut buf).unwrap(); + unsafe { self.clogger.write(&buf[0..sz]); } + } + } + + fn flush(&self) { + unsafe { self.clogger.flush(); } + } +} + +/// Shim is what gets called by the log crate. It holds the config, +/// creates PerThreadLogs on demand, and holds a reference to all +/// the thread local loggers. +struct Shim { + tls: CachedThreadLocal>>, + cfg: LogConfig, +} + +impl Shim { + fn get_per_thread(&self) -> super::Result<&RefCell>> { + self.tls.get_or_try(|| + PerThreadLog::for_current(&self.cfg) + .map(|ptl| Box::new(RefCell::new(Some(ptl))) ) + ) + } + + fn new(cfg: LogConfig) -> Self { + Shim { cfg, tls: CachedThreadLocal::new() } + } + + fn shutdown(&mut self) { + for cell in self.tls.iter_mut() { + if let Some(ptl) = cell.replace(None) { + ptl.flush(); + drop(ptl); + } + } + } + + #[inline] + fn borrow_and_call(&self, f: F) -> Option + where F: FnOnce(&PerThreadLog) + { + self.get_per_thread() + .map(|cell| { + if let Some(ptl) = &*cell.borrow() { + f(ptl); + } + }) + .err() + } +} + +impl Log for Shim { + fn enabled(&self, _: &Metadata) -> bool { + true + } + + fn log(&self, record: &Record) { + if let Some(err) = self.borrow_and_call(|ptl| ptl.log(record)) { + eprintln!("err in Shim::log {:#?}", err); + } + } + + fn flush(&self) { + if let Some(err) = self.borrow_and_call(|ptl| ptl.flush()) { + eprintln!("err in Shim::flush {:#?}", err); + } + } +} + +/// This is the Log instance we give to the log crate. Its job is to +/// hold onto the `Shim` and dispatch calls to it. See `Handle` +/// for a description of the inner structure. +/// +#[doc(hidden)] +struct Logger(Arc>>); + +impl Log for Logger { + fn enabled(&self, metadata: &Metadata) -> bool { + if let Some(n) = &*self.0.get() { + n.enabled(metadata) + } else { + false + } + } + + fn log(&self, record: &Record) { + if let Some(log) = &*self.0.get() { + log.log(record); + } + } + + fn flush(&self) { + if let Some(log) = &*self.0.get() { + log.flush(); + } + } +} + + +/// This is essentially `Arc->ArcCell->Arc->Option->Shim`. The outermost `Arc` is shared +/// between the log crate and this `Handle` that +/// we return to the user to allow them to shut down. +/// +/// ```ignore +/// +-------------------------------+ +/// | | +/// | | +/// +----------+ | +/// | Arc | v +/// | | +--------------------------+ +/// | Logger | | ArcCell | +/// | | | +----------------------+ | +/// | | | | Arc | | +/// +----------+ | | +------------+ | | +/// | | | Option | | | +/// +----------+ | | | +------+ | | | +/// | | | | | | Shim | | | | +/// | | | | | +------+ | | | +/// | Handle | | | +------------+ | | +/// | | | +----------------------+ | +/// | Arc | | | +/// +----------+ +--------------------------+ +/// | ^ +/// | | +/// +-------------------------------+ +/// ``` +/// +/// We perform the shutdown +/// by first swapping out the innermost `Arc` for a no-op (None) version, then unboxing and +/// shutting down the per-thread loggers in the `Shim`. +#[repr(C)] +pub struct Handle { + shim: Arc>> +} + +#[allow(non_camel_case_types)] +#[doc(hidden)] +type log_handle_rs = Handle; + +impl Handle { + fn shutdown(&mut self, timeout: time::Duration) { + let mut active: Arc> = self.shim.set(Arc::new(None)); + + let stop_at = time::SteadyTime::now() + timeout; + + if active.is_none() { + // we've already shut down + eprintln!("already shut down!"); + return; + } + + loop { + if let Some(opt_shim) = Arc::get_mut(&mut active) { + if let Some(shim) = opt_shim { + shim.shutdown(); + break + } + } else { + eprintln!("failed to get_mut on the active logger"); + thread::yield_now(); + } + + if time::SteadyTime::now() < stop_at { + eprintln!("timed out waiting on log shutdown, best of luck!"); + break + } + } + } + + fn is_setup(&self) -> bool { + self.shim.get().is_some() + } +} + +#[no_mangle] +pub unsafe extern "C" fn log_is_setup_rs(cfgp: *mut Handle) -> bool { + ptrs::lift_to_option(cfgp) + .map(|p| (*p).is_setup() ) + .expect("log_is_setup_rs was passed a raw pointer") +} + +const SHUTDOWN_TIMEOUT_MS: u64 = 1000; + +impl Drop for Handle { + fn drop(&mut self) { + self.shutdown(time::Duration::milliseconds(SHUTDOWN_TIMEOUT_MS as i64)); + } +} + +fn log_setup_safe(config: LogConfig) -> Result { + rslog::set_max_level(config.level.to_level_filter()); + let shim = Shim::new(config); + let logger = Logger(Arc::new(ArcCell::new(Arc::new(Some(shim))))); + + let handle = Handle {shim: logger.0.clone()}; + + rslog::set_boxed_logger(Box::new(logger)) + .map(|()| handle) + .map_err(|e| e.into()) +} + +#[no_mangle] +pub unsafe extern "C" fn log_create_handle_rs(cfgp: *mut bind::log_config_rs) -> *mut Handle { + ptrs::null_check(cfgp) // make sure our input is good + .map_err(|e| e.into()) // error type bookkeeping + .and_then(|c|LogConfig::from_raw(c)) // convert the *mut into a rust struct + .and_then(log_setup_safe) // register our logger + .map(|handle| Box::into_raw(Box::new(handle))) // convert our handle into a raw pointer + .unwrap_or_else(|err| { // hand it back to C + eprintln!("ERROR log_create_handle: {:#?}", err); + ptr::null_mut() // unless there was an error, then return NULL + }) +} + +#[no_mangle] +pub unsafe extern "C" fn log_shutdown_rs(ph: *mut Handle, timeout_ms: u32) -> LoggerStatus { + let mut handle = + match ptrs::lift_to_option(ph) { + Some(ph) => Box::from_raw(ph), + None => return LoggerStatus::NullPointerError, + }; + + Handle::shutdown(&mut handle, time::Duration::milliseconds(timeout_ms as i64)); + + LoggerStatus::OK +} + +#[no_mangle] +pub unsafe extern "C" fn log_destroy_handle_rs(pph: *mut *mut Handle) { + assert!(!pph.is_null()); + let ph = *pph; + drop(Box::from_raw(ph)); + *pph = ptr::null_mut(); +} + +// for integration testing with C +#[doc(hidden)] +#[no_mangle] +pub unsafe extern "C" fn log_test_threaded_writes_rs() -> bool { + let t1 = thread::spawn(move || { + for x in 0..10 { + error!("thread 1: {}", x); + } + }); + + let t2 = thread::spawn(move || { + for x in 0..10 { + warn!("thread 2: {}", x); + } + }); + + t1.join().unwrap(); + t2.join().unwrap(); + + true +} + +#[cfg(test)] +mod test { + use std::fs; + use std::sync::mpsc; + use super::*; + use tempfile; + use time; + + + // this is necessary until https://github.com/rust-lang/rust/issues/48854 + // lands in stable + fn assert_result(f: F) + where F: FnOnce() -> Result + { + match f() { + Ok(_) => (), + Err(e) => panic!(e) + } + } + + fn basic_mt_roundtrip() { + assert_result(|| { + let mut stats = LogMetrics::new(); + unsafe { bind::log_setup(stats.as_mut_ptr()) }; + let tmpdir = tempfile::tempdir()?; + + let cfg = LogConfig { + path: tmpdir.path().to_path_buf().to_str().unwrap().to_owned(), + prefix: String::from("testmt"), + buf_size: 0, + level: Level::Trace, + }; + + let handle = log_setup_safe(cfg).unwrap(); + + let t1 = thread::spawn(move || { + error!("thread 1 error"); + }); + + let t2 = thread::spawn(move || { + warn!("thread 2 error"); + }); + + t1.join().unwrap(); + t2.join().unwrap(); + + drop(handle); + + Ok(()) + }) + } + + + fn build(name: &str) -> thread::Builder { + thread::Builder::new().name(name.to_owned()) + } + + fn named_threads_test() { + assert_result(||{ + let mut stats = LogMetrics::new(); + unsafe { bind::log_setup(stats.as_mut_ptr()) }; + let tmpdir = tempfile::tempdir()?; + + let cfg = LogConfig { + path: tmpdir.path().to_path_buf().to_str().unwrap().to_owned(), + prefix: String::from("testmt"), + buf_size: 0, + level: Level::Trace, + }; + + let handle = log_setup_safe(cfg).unwrap(); + + let t1 = build("d_level").spawn(move || { + debug!("debug message"); + }).unwrap(); + + let t2 = build("w_level").spawn(move || { + warn!("warn message"); + }).unwrap(); + + t1.join().unwrap(); + t2.join().unwrap(); + + drop(handle); + + { + let mut dlevelp = tmpdir.path().to_owned(); + dlevelp.push("testmt.d_level.log"); + let md = fs::metadata(dlevelp)?; + assert!(md.len() > 0); + } + + { + let mut wlevelp = tmpdir.path().to_owned(); + wlevelp.push("testmt.w_level.log"); + let md = fs::metadata(wlevelp)?; + assert!(md.len() > 0); + } + + Ok(()) + }) + } + + fn mt_shutdown_resilience_test() { + assert_result(||{ + // make sure a thread logging doesn't crash if we shutdown simultaneously + let mut stats = LogMetrics::new(); + unsafe { bind::log_setup(stats.as_mut_ptr()) }; + let tmpdir = tempfile::tempdir()?; + + let cfg = LogConfig { + path: tmpdir.path().to_path_buf().to_str().unwrap().to_owned(), + prefix: String::from("testmt"), + buf_size: 0, + level: Level::Trace, + }; + + let handle = log_setup_safe(cfg).unwrap(); + + let (start_tx, start_rx) = mpsc::sync_channel::(0); + let (stop_tx, stop_rx) = mpsc::sync_channel::(0); + let (loop_tx, loop_rx) = mpsc::sync_channel::(300); + + eprintln!("start thread"); + let th = build("worker").spawn(move||{ + eprintln!("thread started, waiting for message"); + let msg = start_rx.try_recv().unwrap(); + eprintln!("got start msg: {}", msg); + + let mut count: u64 = 0; + loop { + let tm = time::now_utc(); + trace!("{:#?}", tm.to_timespec()); + count += 1; + loop_tx.send(count).unwrap(); + + match stop_rx.try_recv() { + Ok(_) => { + eprintln!("received stop signal"); + break; + }, + Err(mpsc::TryRecvError::Disconnected) => { + eprintln!("gah! disconnected!"); + panic!("bad things!"); + }, + Err(mpsc::TryRecvError::Empty) => () + }; + } + + eprintln!("while loop exited"); + count + }).unwrap(); + + start_tx.send("GO!".to_owned())?; + + let delay = ::std::time::Duration::from_millis(100); + + assert_eq!(loop_rx.recv_timeout(delay)?, 1); + + eprintln!("dropping handle"); + drop(handle); + + // make sure the thread writes another message or two + assert_eq!(loop_rx.recv_timeout(delay)?, 2); + assert_eq!(loop_rx.recv_timeout(delay)?, 3); + + // signal to the thread that it should exit + stop_tx.send(true)?; + + eprintln!("joining"); + let count = th.join().unwrap(); + eprintln!("thread joined, wrote {} messages", count); + + Ok(()) + }) + } + + // runs this test with process isolation + rusty_fork_test! { + #[test] + fn test_basic_mt_roundtrip() { basic_mt_roundtrip(); } + } + + rusty_fork_test! { + #[test] + fn test_named_threads() { named_threads_test(); } + } + + rusty_fork_test! { + #[test] + fn test_shutdown_resilience() { mt_shutdown_resilience_test(); } + } +} + diff --git a/rust/ccommon_rs/src/ptrs.rs b/rust/ccommon_rs/src/ptrs.rs new file mode 100644 index 000000000..11b930f56 --- /dev/null +++ b/rust/ccommon_rs/src/ptrs.rs @@ -0,0 +1,41 @@ +// ccommon - a cache common library. +// Copyright (C) 2018 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::result; +use std::ptr; + +#[derive(Fail, Debug)] +#[fail(display = "Null pointer exception")] +pub struct NullPointerError; + +pub fn lift_to_option(p: *mut T) -> Option<*mut T> { + if p.is_null() { + None + } else { + Some(p) + } +} + +pub fn null_check(p: *mut T) -> result::Result<*mut T, NullPointerError> { + lift_to_option(p).ok_or_else(|| NullPointerError) +} + +pub fn opt_to_null_mut(o: Option<*mut T>) -> *mut T { + match o { + Some(p) => p, + None => ptr::null_mut(), + } +} + diff --git a/rust/ccommon_rs/src/util.rs b/rust/ccommon_rs/src/util.rs new file mode 100644 index 000000000..cefb110bb --- /dev/null +++ b/rust/ccommon_rs/src/util.rs @@ -0,0 +1,43 @@ +// ccommon - a cache common library. +// Copyright (C) 2018 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +use std::ffi::CStr; +use std::fs; +use std::os::raw::c_char; + +/// Recursively removes files and directories under `path` before removing `path` itself. +/// Returns 0 on success and -1 on error. `errno` will be set to the cause of the failure. +#[no_mangle] +pub unsafe extern "C" fn cc_util_rm_rf_rs(path: *const c_char) -> i32 { + assert!(!path.is_null()); + + let s = + match CStr::from_ptr(path as *mut c_char).to_str() { + Ok(s) => s, + Err(err) => { + eprintln!("ERROR: cc_util_rm_rf_rs: {:#?}", err); + return -1 + } + }; + + match fs::remove_dir_all(s) { + Ok(()) => 0, + Err(err) => { + eprintln!("ERROR, cc_util_rm_rf_rs {:#?}", err); + -1 + } + } +} diff --git a/rust/scripts/setup.sh b/rust/scripts/setup.sh new file mode 100755 index 000000000..259c2a06b --- /dev/null +++ b/rust/scripts/setup.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# utility script run by cmake for writing a .cargo/config that points +# to the common 'target' directory under the CMAKE_BINARY_DIR. This +# allows automated tools (such as the intellij rust plugin or vcode +# rust integration) to share output and avoid recompiling between +# the command line and the IDE. +# +# it is assumed that this script is run with the CWD being the +# place where the .cargo dir should be created. + +set -euo pipefail +IFS=$'\n\t' + +die() { echo "fatal: $*" >&2; exit 1; } + +if [[ -z "${CMAKE_BINARY_DIR:-}" ]]; then + die "CMAKE_BINARY_DIR must be set!" +fi + +mkdir -p .cargo + +cleanup() { + [[ -n "${TEMPFILE:-}" ]] && rm -rf "$TEMPFILE" +} +trap cleanup EXIT + +TEMPFILE="$(mktemp '.cargo/config.XXXXXXXX')" || die "could not create tempfile" + +cat > "$TEMPFILE" <rpos - (*buf)->begin; - uint32_t woffset = (*buf)->wpos - (*buf)->begin; + uint32_t osize, roffset, woffset; + + if (nsize > max_size) { + return CC_ERROR; + } + + osize = buf_size(*buf); + roffset = (*buf)->rpos - (*buf)->begin; + woffset = (*buf)->wpos - (*buf)->begin; nbuf = cc_realloc(*buf, nsize); if (nbuf == NULL) { /* realloc failed, but *buf is still valid */ @@ -80,10 +86,6 @@ dbuf_double(struct buf **buf) rstatus_i status; uint32_t nsize = buf_size(*buf) * 2; - if (nsize > max_size) { - return CC_ERROR; - } - status = _dbuf_resize(buf, nsize); if (status == CC_OK) { INCR(dbuf_metrics, dbuf_double); @@ -100,12 +102,14 @@ dbuf_fit(struct buf **buf, uint32_t cap) rstatus_i status = CC_OK; uint32_t nsize = buf_init_size; - buf_lshift(*buf); - if (buf_rsize(*buf) > cap || cap + BUF_HDR_SIZE > max_size) { + /* check if new cap can contain unread bytes */ + if (buf_rsize(*buf) > cap) { return CC_ERROR; } - /* cap is checked, given how max_size is initialized this is safe */ + buf_lshift(*buf); + + /* double size of buf until it can fit cap */ while (nsize < cap + BUF_HDR_SIZE) { nsize *= 2; } @@ -125,9 +129,9 @@ dbuf_fit(struct buf **buf, uint32_t cap) rstatus_i dbuf_shrink(struct buf **buf) { - rstatus_i status = CC_OK; uint32_t nsize = buf_init_size; uint32_t cap = buf_rsize(*buf); + rstatus_i status = CC_OK; buf_lshift(*buf); @@ -136,7 +140,12 @@ dbuf_shrink(struct buf **buf) } if (nsize != buf_size(*buf)) { + /* + * realloc is not guaranteed to succeed even on trim, but in the case + * that it fails, original buf will still be valid. + */ status = _dbuf_resize(buf, nsize); + if (status == CC_OK) { INCR(dbuf_metrics, dbuf_shrink); } else { diff --git a/src/cc_array.c b/src/cc_array.c index 5768ddde6..0832647a1 100644 --- a/src/cc_array.c +++ b/src/cc_array.c @@ -74,7 +74,7 @@ array_create(struct array **arr, uint32_t nalloc, size_t size) ASSERT(nalloc != 0 && size != 0); *arr = (struct array *)cc_alloc(sizeof(**arr)); - if (arr == NULL) { + if (*arr == NULL) { log_info("array creation failed due to OOM"); return CC_ENOMEM; diff --git a/src/cc_bstring.c b/src/cc_bstring.c index 58f40ca24..68374423c 100644 --- a/src/cc_bstring.c +++ b/src/cc_bstring.c @@ -33,9 +33,9 @@ * raw sequence of character bytes - bstring_copy(). Such String's must be * freed using bstring_deinit() * - * We can also create String as reference to raw string - bstring_set_raw() - * or to string literal - bstring_set_text() or bstring(). Such bstrings don't - * have to be freed. + * We can also create String as reference to C string - bstring_set_cstr() + * or to string literal - bstring_set_literal() or bstring(). Such bstrings + * don't have to be freed. */ void @@ -113,6 +113,46 @@ bstring_compare(const struct bstring *s1, const struct bstring *s2) return cc_bcmp(s1->data, s2->data, s1->len); } +rstatus_i +bstring_atoi64(int64_t *i64, struct bstring *str) +{ + uint32_t offset = 0; + uint8_t c; + int64_t sign = 1; + + if (str->len == 0 || str->len >= CC_INT64_MAXLEN) { + return CC_ERROR; + } + + if (*str->data == '-') { + offset = 1; + sign = -1; + } + + for (*i64 = 0LL; offset < str->len; offset++) { + c = *(str->data + offset); + if (c < '0' || c > '9') { + return CC_ERROR; + } + + // overflow check + if (offset == CC_INT64_MAXLEN - 2) { + if (sign < 0 && *i64 == INT64_MIN / 10 && + c - '0' > (uint64_t)(-INT64_MIN) % 10) { + return CC_ERROR; + } + if (sign > 0 && *i64 == INT64_MAX / 10 && + c - '0' > INT64_MAX % 10) { + return CC_ERROR; + } + } + + *i64 = *i64 * 10LL + sign * (int64_t)(c - '0'); + } + + return CC_OK; +} + rstatus_i bstring_atou64(uint64_t *u64, struct bstring *str) { diff --git a/src/cc_debug.c b/src/cc_debug.c index 7c766260b..c0fd85a78 100644 --- a/src/cc_debug.c +++ b/src/cc_debug.c @@ -20,7 +20,6 @@ #include #include #include -#include