CDB front and backend

.gitignore

@@ -40,3 +40,10 @@ lcov
 
 # python compiled files
 *.pyc
+
+cdb.rs
+*.cdb
+compile_commands.json
+
+*.cmd
+core

.travis.yml

@@ -1,4 +1,4 @@
-sudo: required
+sudo: false
 language: c
 
 # using anchor to import sources into linux builds
@@ -10,6 +10,11 @@ addons:
       - llvm-toolchain-precise-3.7
       - llvm-toolchain-precise
 
+# important for allowed-to-fail matching
+# see https://docs.travis-ci.com/user/customizing-the-build#Rows-that-are-Allowed-to-Fail
+env:
+  - ALLOWED_TO_FAIL=0
+
 # travis currently does not support directly setting gcc/clang with versions
 # (e.g. gcc-4.8) as value for the compiler key. So we will have to manually
 # request these packages and use environment varibles to create the matrix.
@@ -19,39 +24,55 @@ addons:
 matrix:
   include:
     # gcc 4.8 on linux
-    - env: C_COMPILER=gcc-4.8
+    - env:
+        - C_COMPILER=gcc-4.8
       addons:
         apt:
           <<: *apt
-          packages: gcc-4.8
+          packages:
+            - gcc-4.8
+            - libsubunit-dev
+
 
     # gcc 4.9 on linux
-    - env: C_COMPILER=gcc-4.9
+    - env:
+        - C_COMPILER=gcc-4.9
       addons:
         apt:
           <<: *apt
-          packages: gcc-4.9
+          packages:
+            - gcc-4.9
+            - libsubunit-dev
 
     # gcc 5 on linux
-    - env: C_COMPILER=gcc-5
+    - env:
+        - C_COMPILER=gcc-5
       addons:
         apt:
           <<: *apt
-          packages: gcc-5
+          packages:
+            - gcc-5
+            - libsubunit-dev
 
     # clang 3.6 on linux
-    - env: C_COMPILER=clang-3.6
+    - env:
+        - C_COMPILER=clang-3.6
       addons:
         apt:
           <<: *apt
-          packages: clang-3.6
+          packages:
+            - clang-3.6
+            - libsubunit-dev
 
     # clang 3.7 on linux
-    - env: C_COMPILER=clang-3.7
+    - env:
+        - C_COMPILER=clang-3.7
       addons:
         apt:
           <<: *apt
-          packages: clang-3.7
+          packages:
+            - clang-3.7
+            - libsubunit-dev
 
     ## gcc 4.8 on osx
     #- os: osx
@@ -65,38 +86,39 @@ matrix:
     #- os: osx
     #  env: FORMULA=gcc5 COMPILER=gcc C_COMPILER=gcc-5
 
-    # clang 3.6 on osx
+    # OSX 10.13
+    # Apple LLVM version 9.1.0 (clang-902.0.39.2)
+    # Target: x86_64-apple-darwin17.6.0
     - os: osx
-      osx_image: xcode6.4
-      env: C_COMPILER=clang
+      osx_image: xcode9.4
+      env:
+        - C_COMPILER=clang
+        - ALLOWED_TO_FAIL=1
 
-    # clang 3.7 on osx
+    # OSX 10.12
+    # Apple LLVM version 9.0.0 (clang-900.0.39.2)
+    # Target: x86_64-apple-darwin16.7.0
     - os: osx
-      osx_image: xcode7.3
-      env: C_COMPILER=clang
+      osx_image: xcode9.2
+      env:
+        - C_COMPILER=clang
+        - ALLOWED_TO_FAIL=1
 
-          #    # clang 4.2 on osx
-          #    - os: osx
-          #      osx_image: xcode8.2
-          #      env: C_COMPILER=clang
+  allow_failures:
+    - os: osx
+      osx_image: xcode9.4
+      env:
+        - C_COMPILER=clang
+        - ALLOWED_TO_FAIL=1
 
+    - os: osx
+      osx_image: xcode9.2
+      env:
+        - C_COMPILER=clang
+        - ALLOWED_TO_FAIL=1
 
 before_install:
-  # for osx: 0. update brew; 1. install cmake if missing; 2. (gcc) unlink pre-installed gcc; 3. (gcc) install desired version of gcc
-  - 'if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update; fi'
-  - 'if [[ "$TRAVIS_OS_NAME" == "osx" && -z "$(which cmake)" ]]; then brew install cmake; fi' # xcode 7.1 is missing cmake
-  - 'if [[ "$TRAVIS_OS_NAME" == "osx" && "$COMPILER" == "gcc" ]]; then brew unlink gcc || true; fi' # ignore unlink errors
-  - 'if [[ "$TRAVIS_OS_NAME" == "osx" && "$COMPILER" == "gcc" ]]; then brew unlink $FORMULA || true; fi' # ignore unlink errors
-  - 'if [[ "$TRAVIS_OS_NAME" == "osx" && "$COMPILER" == "gcc" ]]; then brew install $FORMULA; fi'
-  - export CC=$C_COMPILER
-  - wget https://github.com/libcheck/check/releases/download/0.11.0/check-0.11.0.tar.gz
-  - tar xvfz check-0.11.0.tar.gz
-  - cd check-0.11.0 && ./configure && make && sudo make install && cd ..
+  - ./ci/before-install.sh
 
 script:
-  - mkdir _build && cd _build
-  - cmake ..
-  - make -j
-  - make check
-  - egrep -r ":F:|:E:" . || true
-  - cd ../test/integration && python test_twemcache.py
+  - ./ci/run.sh

CMakeLists.txt

@@ -39,9 +39,12 @@ option(TARGET_REDIS "build redis binary" ON)
 option(TARGET_SLIMREDIS "build slimredis binary" ON)
 option(TARGET_SLIMCACHE "build slimcache binary" ON)
 option(TARGET_TWEMCACHE "build twemcache binary" ON)
-
+option(TARGET_CDB "build cdb binary" OFF)
 option(TARGET_RESPCLI "build resp-cli binary" ON)
 
+option(RUST_USE_MUSL "build rust deps against musl" OFF)
+option(BUILD_AND_INSTALL_CHECK "build our own version of check and link against it" OFF)
+
 option(COVERAGE "code coverage" OFF)
 
 # Note: duplicate custom targets only works with Makefile generators, will break XCode & VS
@@ -80,6 +83,28 @@ set(CFLAGS_LIST
     "-Wunused-function -Wunused-value -Wunused-variable "
     "-fno-strict-aliasing ")
 
+if(BUILD_AND_INSTALL_CHECK)
+    # (simms) What follows is a crime against build systems as we run the build/install
+    # for the check library up front, during the planning phase.
+
+    set(LIBCHECK_PREFIX "${CMAKE_BINARY_DIR}/check")
+
+    # check for a local install of check
+    if(NOT EXISTS "${LIBCHECK_PREFIX}")
+        # (simms) This is terrible and I did it this way to ensure this gets built
+        # before the rest of the 'check' tests run. This should be rewritten so that
+        # the other dependencies know that there's a target that can build check
+        execute_process(
+            COMMAND "bash" "${PROJECT_SOURCE_DIR}/ci/install-check.sh" "${LIBCHECK_PREFIX}"
+            TIMEOUT 300  # if this doesn't build in 5 minutes something is hosed
+        )
+    endif()
+
+    set(CHECK_ROOT_DIR "${LIBCHECK_PREFIX}")
+    set(CMAKE_REQUIRED_INCLUDES "${CHECK_ROOT_DIR}/include")    # these make check link correctly in ccommon and pelikan
+    set(CMAKE_REQUIRED_LIBRARIES "${CHECK_ROOT_DIR}/lib")
+endif()
+
 if(${OS_PLATFORM} MATCHES "OS_LINUX")
   set(CFLAGS_LIST "${CFLAGS_LIST} -lrt")
 endif()
@@ -96,9 +121,11 @@ endif(COVERAGE)
 # dependency: libccommon
 set(CCOMMON_SOURCE_DIR "${PROJECT_SOURCE_DIR}/deps/ccommon" CACHE PATH "Path to the ccommon")
 add_subdirectory(${CCOMMON_SOURCE_DIR} ${PROJECT_BINARY_DIR}/ccommon)
+
 # other dependencies
 include(FindPackageHandleStandardArgs)
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PROJECT_SOURCE_DIR}/cmake")
+
 find_package(Check)
 if(NOT CHECK_FOUND)
     message(WARNING "Check is required to build and run tests")
@@ -112,6 +139,9 @@ endif(CHECK_FOUND)
 
 find_package(Threads)
 
+enable_language(Rust)
+include(CMakeCargo)
+
 #set(CMAKE_INCLUDE_CURRENT_DIR)
 include_directories(${include_directories}
     "${PROJECT_BINARY_DIR}"
@@ -138,5 +168,15 @@ message(STATUS "HAVE_SIGNAME: " ${HAVE_SIGNAME})
 message(STATUS "HAVE_BACKTRACE: " ${HAVE_BACKTRACE})
 message(STATUS "HAVE_BIG_ENDIAN: " ${HAVE_BIG_ENDIAN})
 
+if(DUMP_ALL)
+    message(STATUS "<<++=====------------------\\/------------------=====++>>")
+    get_cmake_property(_variableNames VARIABLES)
+    foreach (_variableName ${_variableNames})
+        message(STATUS "${_variableName}=${${_variableName}}")
+    endforeach()
+    message(STATUS "<<++=====------------------/\\------------------=====++>>")
+endif()
+
 # Note: to uninstall targets, run:
 #  xargs rm < install_manifest.txt
+# vim:ts=4:sw=4:et

ci/.gitignore

@@ -0,0 +1 @@
+CMAKE_BINARY_DIR

ci/before-install.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -euo pipefail
+IFS=$'\n\t'
+
+die() { echo "fatal: $*" >&2; exit 1; }
+
+TEMP="$(mktemp -d -t TEMP.XXXXXXX)" || die "failed to make tmpdir"
+cleanup() { [[ -n "${TEMP:-}" ]] && rm -rf "${TEMP}"; }
+trap cleanup EXIT
+
+TOPLEVEL="$(git -C "$(cd "$(dirname "$0")" >/dev/null || exit 1; pwd)" rev-parse --show-toplevel)" || die 'failed to find TOPLEVEL'
+
+# for osx: 0. update brew; 1. install cmake if missing; 2. (gcc) unlink pre-installed gcc; 3. (gcc) install desired version of gcc
+
+if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+  brew update &>/dev/null
+  brew install cmake || true            # xcode 8.1 is missing cmake
+
+  if [[ "$C_COMPILER" =~ ^gcc && -n "${FORMULA:-}" ]]; then
+    brew unlink gcc || true
+    brew unlink "$FORMULA" || true
+    brew install "$FORMULA"
+  fi
+fi
+
+export CC="$C_COMPILER"
+
+curl https://sh.rustup.rs -sSf | sh -s -- -y

ci/install-check.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -euo pipefail
+IFS=$'\n\t'
+
+die() { echo "fatal: $*" >&2; exit 1; }
+
+if [[ $# -lt 1 ]]; then
+  echo "Usage: $0 check-install-path"
+  exit 1
+fi
+
+CHECK_PREFIX="$1"
+shift
+
+TEMP="$(mktemp -d -t TEMP.XXXXXXX)" || die "failed to make tmpdir"
+cleanup() { [[ -n "${TEMP:-}" ]] && rm -rf "${TEMP}"; }
+trap cleanup EXIT
+
+TOPLEVEL="$(git -C "$(cd "$(dirname "$0")" >/dev/null || exit 1; pwd)" rev-parse --show-toplevel)" || die 'failed to find TOPLEVEL'
+
+CHECK_VERSION=0.12.0
+CHECK_TARBALL="check-${CHECK_VERSION}.tar.gz"
+CHECK_DIR="check-${CHECK_VERSION}"
+
+(
+  cd "$TEMP" &&
+    wget "https://github.com/libcheck/check/releases/download/${CHECK_VERSION}/${CHECK_TARBALL}" &&
+    tar xvfz "${CHECK_TARBALL}" &&
+    cd "${CHECK_DIR}" &&
+    ./configure --prefix="$CHECK_PREFIX" &&
+    make &&
+    make install
+) || die "check build failed"

ci/local-testing-with-docker.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+set -euo pipefail
+IFS=$'\n\t'
+
+die() { echo "fatal: $*" >&2; exit 1; }
+
+TEMP="$(mktemp -d -t TEMP.XXXXXXX)" || die "failed to make tmpdir"
+cleanup() { [[ -n "${TEMP:-}" ]] && rm -rf "${TEMP}"; }
+trap cleanup EXIT
+
+# This script installs a travis-ci docker image and attempts to run the build
+# script in that container. in order for it to work you have to have the 'travis'
+# command-line utility installed and the 'travis-build' plugin installed as well.
+#
+# instructions here: https://github.com/travis-ci/travis-build#use-as-addon-for-cli
+
+CONTAINER_NAME='pelikan-travis-debug'
+
+# this will undoubtedly need to be updated regularly
+# see https://stackoverflow.com/a/49019950/965434 for more context
+#
+INSTANCE='travisci/ci-garnet:packer-1512502276-986baf0'
+
+
+echo "cleanup previous run's container, no problem if failure" >&2
+
+docker stop "$CONTAINER_NAME" || true
+docker rm "$CONTAINER_NAME" || true
+
+set -x
+
+docker run --name "$CONTAINER_NAME" -dit "$INSTANCE" /sbin/init
+
+CI_SCRIPT="$TEMP/ci.sh"
+
+"$HOME/.travis/travis-build/bin/travis" compile > "$CI_SCRIPT"
+chmod 755 "$CI_SCRIPT"
+
+docker cp "$CI_SCRIPT" "$CONTAINER_NAME:/home/travis/ci.sh"
+
+cat <<EO_RUN_SCRIPT >"$TEMP/run.sh"
+#!/bin/bash
+
+set -euo pipefail
+IFS=$'\n\t'
+
+su - travis -- -c 'cd /home/travis && ./ci.sh'
+
+EO_RUN_SCRIPT
+
+docker cp "$TEMP/run.sh" "$CONTAINER_NAME:/run.sh"
+
+docker exec "$CONTAINER_NAME" "/bin/bash" "/run.sh"