From f55e736eb544a8012f43a764627736fa66f4dbc4 Mon Sep 17 00:00:00 2001 From: Yao Yue Date: Fri, 22 May 2020 18:00:10 -0700 Subject: [PATCH] adding a doubling function to slab hash table for offline use; adding metrics (#289) * adding metrics to the hash module * Squashed 'deps/ccommon/' changes from f5efe29..5e64de2 5e64de2 WIP: add fluxcapacitor support for linux build (#242) 55769cc restore cmake for rust libraries (#241) 6b1d8d5 Improve the Rust build story (#240) 5955d6b move nodelay setting to the server socket, and accepted connections will inherit that (#238) aafd20f formatting (#237) 3b5c069 Update ccommon_rs to use bytes v5.0 (#235) df74087 Address clippy lints in rust code (#234) 7c5bbd1 Make metrics and options Send + Sync (#233) c2e617e Ensure that manually implemented C options use the correct name (#232) 6e76741 Add some docs b5b4c5a Change check_pipe to use nanosleep instead of usleep (#231) eb0a389 Use name of field instead of description for C metrics (#230) 16ddc76 Fix broken buf impls (#229) eefcdcb Avoid redundant rebuilds of rust packages in CI (#228) 33f62a8 Update bindgen to also generate bindings recursively (#227) 27ffc7c Implement bytes::Buf and Bytes::ButMut on Buf and OwnedBuf (#226) f873930 Various small bugfixes and usability improvements for rust code (#225) 37a1ecd Port option parsing module to Rust (#224) 38f7556 Fix failed test detection (#215) 0ab1604 Conditionally use std::any::type_name if it's supported (#223) 98176d3 Backport changes from twitter/pelikan#265 (#222) ba54096 Remove test for removed rust logging functionality (#221) 519118d Rewrite cmake cargo build wrapper (#220) 5d23b3a Fix some small typos found in twitter/pelikan#263 (#219) 475dda7 Clean up logging shim (#218) 1d28dd2 Expand rust bindings to add options, metrics, and ccbox (#217) 88b5400 Upstream changes from twitter/pelikan#261 (#216) 4e99e63 fix a bug and change how check is found (#214) 54067ef slightly simplify accept error-handling logic (#210) e9fe980 Fix synchronize ccommon with pelikan deps/ccommon (#212) 7eb6424 Cleanup libcheck related code (#211) 683bc1a cc_bstring simplify and fix (#207) 8737d99 continue on server socket on non-blocking errors (#209) 2a62281 add atoi64 to bstring (#206) f71c657 cc_option simplify _allowed_in_name (#205) 24e3131 Add ITT instrumentation option (#204) 236c98d Fix docs (#200) e58f6a8 cc_array and cc_ring_array NULL fixes (#201) 1c8df42 Add basic support of build type (#199) 7107988 Fix now_ns() (#198) da240e5 cc: extend cc_util module (#196) 4846b15 Fix TAILQ_REINIT (#195) 4f5dbb0 Update Cmake version to 2.8 (#197) 2e6f78a cc_mm use OS_DARWIN macro to detect OS (#194) 57acaf6 cc: extend queue module (#193) a64ada2 cc: extend duration module (#192) b117632 reverting CMake file changes (#191) dea5bee backport changes made to ccommon in pelikan (#190) a4c0334 add linebreak to stats_log() (#188) 05eb03e fix inconsistent naming and bump version (#187) 4acc53a Stats to file (#186) 2168fec minimize osx build config (#185) 42b24de Simplify rust options, specify fewer output targets (#183) c9fa905 update CMakeRust used to latest version, tweaks to make build work (#184) 2ef0163 Reorder dependency includes in cmake, don't parallel build (#182) a6a54d9 remove endian-specific logic from str*cmp (#177) 4c0668b epoll_create* ignores size hint in newer kernels, switch to new API (#179) c9c5ee5 improve cc_bstring string literal and cstring names (#176) 0184d73 Add unit tests for buffer, fix buf/dbuf bugs and refactor (#174) d7dab43 create a .cargo/config so intellij uses the same target dir as cmake (#173) e710712 use accept4 for tcp_accept when available (#171) 21ba10e Remove cargo lock for shared lib, closes #169 (#172) 24660f1 update style guide (#170) 17baf1e Per thread logging (#168) git-subtree-dir: deps/ccommon git-subtree-split: 5e64de297dbf1d49b59746431448f5f4dfb5d640 * clean up cmake file * rename findcheck cmake module * adding new metrics to rust wrapper * add ci/cargo.sh to pelikan for consistent invocation * Squashed 'deps/ccommon/' changes from 5e64de2..0db032a 0db032a tuning cmake / ci further (#243) 49dcaff improve cargo build script (#244) git-subtree-dir: deps/ccommon git-subtree-split: 0db032afcac0a75887281c3876ed9f7bb10391ce * update ci based on changes first tested in ccommon --- .travis.yml | 50 ++---- CMakeLists.txt | 155 +++++++++++++----- ci/cargo.sh | 6 + ci/install-check.sh | 34 +++- ci/install-fluxcapacitor.sh | 55 +++++++ cmake/{FindCheck.cmake => FindCHECK.cmake} | 0 deps/ccommon/.travis.yml | 41 ++--- deps/ccommon/CMakeLists.txt | 152 ++++++++++------- deps/ccommon/ci/cargo.sh | 3 +- deps/ccommon/ci/install-check.sh | 17 +- deps/ccommon/ci/install-fluxcapacitor.sh | 55 +++++++ deps/ccommon/ci/run.sh | 1 - .../{FindCheck.cmake => FindCHECK.cmake} | 0 deps/ccommon/test/channel/pipe/check_pipe.c | 7 +- deps/ccommon/test/time/wheel/check_wheel.c | 21 ++- src/rust-util/pelikan-sys/src/storage/slab.rs | 6 +- src/storage/slab/hashtable.c | 55 ++++++- src/storage/slab/hashtable.h | 5 +- src/storage/slab/slab.c | 2 +- src/storage/slab/slab.h | 6 +- 20 files changed, 476 insertions(+), 195 deletions(-) create mode 100755 ci/cargo.sh create mode 100755 ci/install-fluxcapacitor.sh rename cmake/{FindCheck.cmake => FindCHECK.cmake} (100%) create mode 100755 deps/ccommon/ci/install-fluxcapacitor.sh rename deps/ccommon/cmake/{FindCheck.cmake => FindCHECK.cmake} (100%) diff --git a/.travis.yml b/.travis.yml index b2eb3f1d4..46c0ecd39 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ sudo: false language: c -dist: xenial +dist: bionic # using anchor to import sources into linux builds addons: @@ -11,69 +11,47 @@ addons: # important for allowed-to-fail matching # see https://docs.travis-ci.com/user/customizing-the-build#Rows-that-are-Allowed-to-Fail env: - - ALLOWED_TO_FAIL=0 + global: + - MAKEFLAGS="-j 2" # travis currently does not support directly setting gcc/clang with versions # (e.g. gcc-4.8) as value for the compiler key. So we will have to manually # request these packages and use environment varibles to create the matrix. -matrix: +jobs: + fast_finish: true include: - - name: "gcc-5 on Linux" + - name: "gcc-7 on Linux" compiler: gcc - addons: - apt: - <<: *apt - packages: - - libsubunit-dev - - name: "gcc-5 on Linux, Rust enabled" + - name: "gcc-7 on Linux, Rust enabled" compiler: gcc env: - RUST_ENABLED=1 - addons: - apt: - <<: *apt - packages: - - libsubunit-dev - name: "cargo build" language: rust - addons: - apt: - <<: *apt - packages: - - libsubunit-dev script: - - cargo build - - cargo test - - cargo build --release - - cargo test --release + - ./ci/cargo.sh - os: osx - osx_image: xcode10.1 + osx_image: xcode11.4 compiler: clang - os: osx - osx_image: xcode10.1 + osx_image: xcode11.4 language: rust script: - - cargo build - - cargo test - - cargo build --release - - cargo test --release + - ./ci/cargo.sh allow_failures: - os: osx - osx_image: xcode10.1 + osx_image: xcode11.4 compiler: clang - os: osx - osx_image: xcode10.1 + osx_image: xcode11.4 language: rust script: - - cargo build - - cargo test - - cargo build --release - - cargo test --release + - ./ci/cargo.sh before_install: - ./ci/before-install.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 0793ead5b..b6ef1810f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,16 @@ -cmake_minimum_required(VERSION 2.8) +cmake_minimum_required(VERSION 3.0) project(pelikan C) -enable_testing() +# Uncomment the following to output dependency graph debugging messages +# set_property(GLOBAL PROPERTY GLOBAL_DEPENDS_DEBUG_MODE 1) + +################### +# detect platform # +################### + +# TODO(yao): +# 1. make this a .cmake macro and put it under cmake/ +# 2. avoid calling this twice when included by another project, e.g. Pelikan # detect platform macro(set_platform system_name) @@ -21,6 +30,10 @@ if(OS_PLATFORM STREQUAL "OS_UNSUPPORTED") message(FATAL_ERROR "unsupported operating system") endif() +#################### +# define variables # +#################### + # the following sections work with config.h(.in): version, compile variables # config.h.in has to include entries set/tested here for them to have effect @@ -28,13 +41,29 @@ endif() set(${PROJECT_NAME}_VERSION_MAJOR 0) set(${PROJECT_NAME}_VERSION_MINOR 1) set(${PROJECT_NAME}_VERSION_PATCH 1) -set(${PROJECT_NAME}_VERSION ${${PROJECT_NAME}_VERSION_MAJOR}.${${PROJECT_NAME}_VERSION_MINOR}.${${PROJECT_NAME}_VERSION_PATCH}) +set(${PROJECT_NAME}_VERSION + ${${PROJECT_NAME}_VERSION_MAJOR}.${${PROJECT_NAME}_VERSION_MINOR}.${${PROJECT_NAME}_VERSION_PATCH} + ) +set(${PROJECT_NAME}_RELEASE_VERSION + ${${PROJECT_NAME}_VERSION_MAJOR}.${${PROJECT_NAME}_VERSION_MINOR} + ) # flags => compile-time variables: use modules/macros option(HAVE_ASSERT_LOG "assert_log enabled by default" ON) option(HAVE_ASSERT_PANIC "assert_panic disabled by default" OFF) option(HAVE_LOGGING "logging enabled by default" ON) option(HAVE_STATS "stats enabled by default" ON) +option(HAVE_TEST "test built by default" ON) +option(HAVE_DEBUG_MM "debugging oriented memory management disabled by default" OFF) +option(HAVE_COVERAGE "code coverage" OFF) +option(HAVE_RUST "rust bindings not built by default" OFF) +option(HAVE_ITT_INSTRUMENTATION "instrument code with ITT API" OFF) + +option(FORCE_CHECK_BUILD "Force building check with ci/install-check.sh" OFF) + +if(HAVE_RUST) + option(RUST_VERBOSE_BUILD "pass -vv to cargo compilation" OFF) +endif() option(TARGET_PINGSERVER "build pingserver binary" ON) option(TARGET_RDS "build rich data server binary" ON) @@ -45,25 +74,8 @@ option(TARGET_CDB "build cdb binary (implies HAVE_RUST)" OFF) option(TARGET_RESPCLI "build resp-cli binary" ON) option(TARGET_HTTP "build experimental twemcache-http server (implies HAVE_RUST)" OFF) -option(HAVE_RUST "build features written in rust" OFF) -option(RUST_USE_MUSL "build rust deps against musl" OFF) -option(BUILD_AND_INSTALL_CHECK "build our own version of check and link against it" OFF) option(USE_PMEM "build persistent memory features" OFF) -option(HAVE_ITT_INSTRUMENTATION "instrument code with ITT API" OFF) - -option(COVERAGE "code coverage" OFF) -# Note: duplicate custom targets only works with Makefile generators, will break XCode & VS -# reference: http://public.kitware.com/Bug/view.php?id=6348 -set_property(GLOBAL PROPERTY ALLOW_DUPLICATE_CUSTOM_TARGETS 1) -set(CMAKE_SKIP_INSTALL_ALL_DEPENDENCY true) -set(CMAKE_MACOSX_RPATH 1) - -# Required for properly linking with rust code -set(CMAKE_POSITION_INDEPENDENT_CODE true) - -include(CheckSymbolExists) -check_symbol_exists(sys_signame signal.h HAVE_SIGNAME) include(CheckFunctionExists) check_function_exists(backtrace HAVE_BACKTRACE) @@ -74,6 +86,15 @@ configure_file( "${PROJECT_SOURCE_DIR}/cmake/config.h.in" "${PROJECT_BINARY_DIR}/config.h") +# Note: duplicate custom targets only works with Makefile generators, will break XCode & VS +# reference: http://public.kitware.com/Bug/view.php?id=6348 +set_property(GLOBAL PROPERTY ALLOW_DUPLICATE_CUSTOM_TARGETS 1) +set(CMAKE_SKIP_INSTALL_ALL_DEPENDENCY true) +set(CMAKE_MACOSX_RPATH 1) + +# Required for properly linking with rust code +set(CMAKE_POSITION_INDEPENDENT_CODE true) + # set compiler flags # string concat is easier in 3.0, but older versions don't have the concat subcommand # so we are using list as input until we move to new version @@ -145,6 +166,48 @@ add_subdirectory(${CCOMMON_SOURCE_DIR} ${PROJECT_BINARY_DIR}/ccommon) include(FindPackageHandleStandardArgs) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PROJECT_SOURCE_DIR}/cmake") +# test dependencies +if (HAVE_TEST) + enable_testing() + # first we try default ways of finding gmodules + find_package(CHECK) + if(CHECK_FOUND) + check_symbol_exists(ck_assert_int_eq check.h CHECK_WORKING) + endif(CHECK_FOUND) + # if we don't have a working version of check, build it + if(NOT CHECK_WORKING OR FORCE_CHECK_BUILD) + set(LIBCHECK_PREFIX "${CMAKE_BINARY_DIR}/check") + execute_process( + COMMAND "bash" "${PROJECT_SOURCE_DIR}/ci/install-check.sh" "${LIBCHECK_PREFIX}" + TIMEOUT 300 # if this doesn't build in 5 minutes something is hosed + RESULT_VARIABLE LIBCHECK_RETCODE + ) + if(LIBCHECK_RETCODE) # non-zero means error + message(STATUS "build libcheck failed, return code: " ${LIBCHECK_RETCODE}) + else(LIBCHECK_RETCODE) + # use locally built libcheck + set(CHECK_ROOT_DIR "${LIBCHECK_PREFIX}") + find_package(CHECK) + endif(LIBCHECK_RETCODE) + endif(NOT CHECK_WORKING OR FORCE_CHECK_BUILD) + + # use fluxcapacitor to mock time + if(OS_PLATFORM STREQUAL "OS_LINUX") + set(FLUXCAP_PREFIX "${CMAKE_BINARY_DIR}/fluxcapacitor") + execute_process( + COMMAND "bash" "${PROJECT_SOURCE_DIR}/ci/install-fluxcapacitor.sh" "${FLUXCAP_PREFIX}" + TIMEOUT 60 # if this doesn't build in 60 seconds something is hosed + RESULT_VARIABLE FLUXCAP_RETCODE + ) + if(FLUXCAP_RETCODE) # non-zero means error + message(STATUS "build fluxcapacitor failed, return code: " ${FLUXCAP_RETCODE}) + else(FLUXCAP_RETCODE) + set(FLUXCAP_BINARY "${FLUXCAP_PREFIX}/fluxcapacitor") + message(STATUS "fluxcapacitor available at: " ${FLUXCAP_BINARY}) + endif(FLUXCAP_RETCODE) + endif(OS_PLATFORM STREQUAL "OS_LINUX") +endif(HAVE_TEST) + find_package(PkgConfig QUIET) if (USE_PMEM) @@ -167,17 +230,6 @@ if (HAVE_ITT_INSTRUMENTATION) link_libraries(${ITTNOTIFY_LIBRARIES}) endif(HAVE_ITT_INSTRUMENTATION) -find_package(Check) -if(NOT CHECK_FOUND) - message(WARNING "Check is required to build and run tests") -endif(NOT CHECK_FOUND) -if(CHECK_FOUND) - check_symbol_exists(ck_assert_int_eq check.h CHECK_WORKING) - if(NOT CHECK_WORKING) - message(WARNING "Check version too old to build tests") - endif(NOT CHECK_WORKING) -endif(CHECK_FOUND) - find_package(Threads) if(TARGET_CDB) @@ -202,30 +254,45 @@ include_directories(${include_directories} # server & (cli) client add_subdirectory(src) -# tests: always build last -if(CHECK_FOUND) - include_directories(${CHECK_INCLUDES}) - link_directories(${CHECK_LIBRARY_DIRS}) +if(HAVE_TEST) + include_directories(${include_directories} ${CHECK_INCLUDES}) add_subdirectory(test) -endif(CHECK_FOUND) + if(${OS_PLATFORM} MATCHES "OS_LINUX") + add_subdirectory(benchmarks) + endif() +endif(HAVE_TEST) -if(${OS_PLATFORM} MATCHES "OS_LINUX") - add_subdirectory(benchmarks) -endif() -# print a summary +################### +# print a summary # +################### +message(STATUS "<<++=====------------------\\/------------------=====++>>") +message(STATUS "<<++ pelikan summary ++>>") +message(STATUS "<<++=====------------------/\\------------------=====++>>") +message(STATUS "=============CMake related=============") message(STATUS "CMAKE_BUILD_TYPE: " ${CMAKE_BUILD_TYPE}) - message(STATUS "PLATFORM: " ${OS_PLATFORM}) - message(STATUS "CPPFLAGS: " ${CMAKE_CPP_FLAGS}) message(STATUS "CFLAGS: " ${CMAKE_C_FLAGS}) +message(STATUS "=======================================") -message(STATUS "HAVE_SIGNAME: " ${HAVE_SIGNAME}) - +message(STATUS "=======Status of system features=======") message(STATUS "HAVE_BACKTRACE: " ${HAVE_BACKTRACE}) +message(STATUS "=======================================") + +message(STATUS "======Status of optional features======") message(STATUS "HAVE_RUST: " ${HAVE_RUST}) +message(STATUS "HAVE_ASSERT_LOG: " ${HAVE_ASSERT_LOG}) +message(STATUS "HAVE_ASSERT_PANIC: " ${HAVE_ASSERT_PANIC}) +message(STATUS "HAVE_LOGGING: " ${HAVE_LOGGING}) +message(STATUS "HAVE_STATS: " ${HAVE_STATS}) +message(STATUS "HAVE_ITT_INSTRUMENTATION: " ${HAVE_ITT_INSTRUMENTATION}) +message(STATUS "HAVE_DEBUG_MM: " ${HAVE_DEBUG_MM}) +message(STATUS "HAVE_TEST: " ${HAVE_TEST}) +message(STATUS "HAVE_COVERAGE: " ${HAVE_COVERAGE}) +message(STATUS "USE_PMEM: " ${USE_PMEM}) +message(STATUS "=======================================") if(DUMP_ALL) message(STATUS "<<++=====------------------\\/------------------=====++>>") diff --git a/ci/cargo.sh b/ci/cargo.sh new file mode 100755 index 000000000..c96ab80d0 --- /dev/null +++ b/ci/cargo.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +cargo build +cargo test +cargo build --release +cargo test --release diff --git a/ci/install-check.sh b/ci/install-check.sh index 4d65f61b4..070c4ebea 100755 --- a/ci/install-check.sh +++ b/ci/install-check.sh @@ -17,20 +17,42 @@ TEMP="$(mktemp -d -t TEMP.XXXXXXX)" || die "failed to make tmpdir" cleanup() { [[ -n "${TEMP:-}" ]] && rm -rf "${TEMP}"; } trap cleanup EXIT -realpath() { python -c "from __future__ import print_function; import os,sys; print(os.path.realpath(sys.argv[1]))" "$1"; } +realpath() { python3 -c "import os,sys; print(os.path.realpath(sys.argv[1]))" "$1"; } TOPLEVEL="$(cd "$(dirname "$(realpath "$0" >/dev/null || exit 1)")" && git rev-parse --show-toplevel)" || die 'failed to find TOPLEVEL' -CHECK_VERSION=0.12.0 + +CHECK_VERSION="0.14.0" CHECK_TARBALL="check-${CHECK_VERSION}.tar.gz" CHECK_DIR="check-${CHECK_VERSION}" +CHECK_LOG="build-check.log" + +echo "building and installing check" >&2 + +echo "building and installing check" >&2 ( cd "$TEMP" && wget "https://github.com/libcheck/check/releases/download/${CHECK_VERSION}/${CHECK_TARBALL}" && - tar xvfz "${CHECK_TARBALL}" && + tar xfz "${CHECK_TARBALL}" && cd "${CHECK_DIR}" && - ./configure --prefix="$CHECK_PREFIX" && - make && + mkdir build && + cd build && + cmake -DCMAKE_INSTALL_PREFIX="${CHECK_PREFIX}" .. && + make -j && make install -) || die "check build failed" +) >$TEMP/${CHECK_LOG} 2>&1 + +RESULT=$? +if [[ $RESULT -ne 0 ]]; then + cat >&2 <&2 +fi + +exit $RESULT diff --git a/ci/install-fluxcapacitor.sh b/ci/install-fluxcapacitor.sh new file mode 100755 index 000000000..643350f63 --- /dev/null +++ b/ci/install-fluxcapacitor.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +set -euo pipefail +IFS=$'\n\t' + +die() { echo "fatal: $*" >&2; exit 1; } + +if [[ $# -lt 1 ]]; then + echo "Usage: $0 fluxcap-install-path" + exit 1 +fi + +FLUXCAP_PREFIX="$1" +shift + +TEMP="$(mktemp -d -t TEMP.XXXXXXX)" || die "failed to make tmpdir" +cleanup() { [[ -n "${TEMP:-}" ]] && rm -rf "${TEMP}"; } +trap cleanup EXIT + +realpath() { python3 -c "import os,sys; print(os.path.realpath(sys.argv[1]))" "$1"; } + +TOPLEVEL="$(cd "$(dirname "$(realpath "$0" >/dev/null || exit 1)")" && git rev-parse --show-toplevel)" || die 'failed to find TOPLEVEL' + + +FLUXCAP_VERSION=0.1 +FLUXCAP_TARBALL="${FLUXCAP_VERSION}.tar.gz" +FLUXCAP_DIR="fluxcapacitor-${FLUXCAP_VERSION}" +FLUXCAP_LOG="build-fluxcapacitor.log" + +echo "building fluxcapacitor" >&2 + +( + cd "$TEMP" && + wget "https://github.com/thinkingfish/fluxcapacitor/archive/${FLUXCAP_TARBALL}" && + tar xfz "${FLUXCAP_TARBALL}" && + pwd && + ls . && + cd "${FLUXCAP_DIR}" && + make build +) >$TEMP/${FLUXCAP_LOG} 2>&1 + +RESULT=$? +if [[ $RESULT -ne 0 ]]; then + cat >&2 <&2 +fi + +exit $RESULT diff --git a/cmake/FindCheck.cmake b/cmake/FindCHECK.cmake similarity index 100% rename from cmake/FindCheck.cmake rename to cmake/FindCHECK.cmake diff --git a/deps/ccommon/.travis.yml b/deps/ccommon/.travis.yml index 0286b3a85..46c0ecd39 100644 --- a/deps/ccommon/.travis.yml +++ b/deps/ccommon/.travis.yml @@ -1,6 +1,6 @@ sudo: false language: c -dist: xenial +dist: bionic # using anchor to import sources into linux builds addons: @@ -11,57 +11,44 @@ addons: # important for allowed-to-fail matching # see https://docs.travis-ci.com/user/customizing-the-build#Rows-that-are-Allowed-to-Fail env: - - ALLOWED_TO_FAIL=0 + global: + - MAKEFLAGS="-j 2" # travis currently does not support directly setting gcc/clang with versions # (e.g. gcc-4.8) as value for the compiler key. So we will have to manually # request these packages and use environment varibles to create the matrix. -matrix: +jobs: + fast_finish: true include: - - name: "gcc-5 on Linux" + - name: "gcc-7 on Linux" compiler: gcc - addons: - apt: - <<: *apt - packages: - - libsubunit-dev - - name: "gcc-5 on Linux, Rust enabled" + - name: "gcc-7 on Linux, Rust enabled" compiler: gcc env: - RUST_ENABLED=1 - addons: - apt: - <<: *apt - packages: - - libsubunit-dev - - - os: osx - osx_image: xcode10.1 - compiler: clang - name: "cargo build" language: rust - addons: - apt: - <<: *apt - packages: - - libsubunit-dev script: - ./ci/cargo.sh - os: osx - osx_image: xcode10.1 + osx_image: xcode11.4 + compiler: clang + + - os: osx + osx_image: xcode11.4 language: rust script: - ./ci/cargo.sh allow_failures: - os: osx - osx_image: xcode10.1 + osx_image: xcode11.4 compiler: clang - os: osx - osx_image: xcode10.1 + osx_image: xcode11.4 language: rust script: - ./ci/cargo.sh diff --git a/deps/ccommon/CMakeLists.txt b/deps/ccommon/CMakeLists.txt index e76d9e3f1..f8eda0f4c 100644 --- a/deps/ccommon/CMakeLists.txt +++ b/deps/ccommon/CMakeLists.txt @@ -1,11 +1,9 @@ -cmake_minimum_required(VERSION 2.8) +cmake_minimum_required(VERSION 3.0) project(ccommon C) # Uncomment the following to output dependency graph debugging messages # set_property(GLOBAL PROPERTY GLOBAL_DEPENDS_DEBUG_MODE 1) -enable_testing() - ################### # detect platform # ################### @@ -54,46 +52,23 @@ option(HAVE_ASSERT_LOG "assert_log enabled by default" ON) option(HAVE_ASSERT_PANIC "assert_panic disabled by default" OFF) option(HAVE_LOGGING "logging enabled by default" ON) option(HAVE_STATS "stats enabled by default" ON) +option(HAVE_TEST "test built by default" ON) option(HAVE_DEBUG_MM "debugging oriented memory management disabled by default" OFF) -option(COVERAGE "code coverage" OFF) +option(HAVE_COVERAGE "code coverage" OFF) option(HAVE_RUST "rust bindings not built by default" OFF) option(HAVE_ITT_INSTRUMENTATION "instrument code with ITT API" OFF) +option(FORCE_CHECK_BUILD "Force building check with ci/install-check.sh" OFF) + if(HAVE_RUST) option(RUST_VERBOSE_BUILD "pass -vv to cargo compilation" OFF) endif() -if(BUILD_AND_INSTALL_CHECK) - # (simms) What follows is a crime against build systems as we run the build/install - # for the check library up front, during the planning phase. - - set(LIBCHECK_PREFIX "${CMAKE_BINARY_DIR}/check") - - # check for a local install of check - if(NOT EXISTS "${LIBCHECK_PREFIX}") - # (simms) This is terrible and I did it this way to ensure this gets built - # before the rest of the 'check' tests run. This should be rewritten so that - # the other dependencies know that there's a target that can build check - execute_process( - COMMAND "bash" "${PROJECT_SOURCE_DIR}/ci/install-check.sh" "${LIBCHECK_PREFIX}" - TIMEOUT 300 # if this doesn't build in 5 minutes something is hosed - ) - endif() - - set(CHECK_ROOT_DIR "${LIBCHECK_PREFIX}") - set(CMAKE_REQUIRED_INCLUDES "${CHECK_ROOT_DIR}/include") # these make check link correctly in ccommon and pelikan -endif() - include(CheckIncludeFiles) if(OS_PLATFORM STREQUAL "OS_LINUX") check_include_files(linux/time64.h HAVE_TIME64) endif() -include(CheckIncludeFiles) -if(OperatingSystem STREQUAL "OS_LINUX") - check_include_files(linux/time64.h HAVE_TIME64) -endif() - include(CheckSymbolExists) check_symbol_exists(sys_signame signal.h HAVE_SIGNAME) @@ -113,11 +88,10 @@ configure_file( ########################## # set compiler flags -# string concat is easier in 3.0, but older versions don't have the concat subcommand -# so we are using list as input until we move to new version + add_definitions(-D_GNU_SOURCE -D_FILE_OFFSET_BITS=64) -# Set a default build type (Release) if none was specified +# Set a default build type (Release) if none was specified if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release) endif() @@ -129,40 +103,63 @@ else() endif() set(CMAKE_MACOSX_RPATH 1) -set(CFLAGS_LIST +string(CONCAT CFLAGS "-std=c11 " "-ggdb3 " "-Wall " "-Wmissing-prototypes -Wmissing-declarations -Wredundant-decls " "-Wunused-function -Wunused-value -Wunused-variable " "-fstrict-aliasing ") -string(REPLACE "" "" CFLAGS ${CFLAGS_LIST}) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CFLAGS}") if(CMAKE_COMPILER_IS_GNUCC) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wl,--no-as-needed -ldl -pthread -fPIC") endif() -if (COVERAGE) - if(NOT ${CMAKE_BUILD_TYPE} MATCHES Debug) - message(WARNING "Code coverage results with an optimised (non-Debug) build may be misleading" ) - endif() - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage") -endif(COVERAGE) - -# test dependencies include(FindPackageHandleStandardArgs) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PROJECT_SOURCE_DIR}/cmake") -find_package(Check) -if(NOT CHECK_FOUND) - message(WARNING "Check is required to build and run tests") -endif(NOT CHECK_FOUND) -if(CHECK_FOUND) - check_symbol_exists(ck_assert_int_eq check.h CHECK_WORKING) - if(NOT CHECK_WORKING) - message(WARNING "Check version too old to build tests") - endif(NOT CHECK_WORKING) -endif(CHECK_FOUND) + +# test dependencies +if (HAVE_TEST) + enable_testing() + # first we try default ways of finding gmodules + find_package(CHECK) + if(CHECK_FOUND) + check_symbol_exists(ck_assert_int_eq check.h CHECK_WORKING) + endif(CHECK_FOUND) + # if we don't have a working version of check, build it + if(NOT CHECK_WORKING OR FORCE_CHECK_BUILD) + set(LIBCHECK_PREFIX "${CMAKE_BINARY_DIR}/check") + execute_process( + COMMAND "bash" "${PROJECT_SOURCE_DIR}/ci/install-check.sh" "${LIBCHECK_PREFIX}" + TIMEOUT 300 # if this doesn't build in 5 minutes something is hosed + RESULT_VARIABLE LIBCHECK_RETCODE + ) + if(LIBCHECK_RETCODE) # non-zero means error + message(STATUS "build libcheck failed, return code: " ${LIBCHECK_RETCODE}) + else(LIBCHECK_RETCODE) + # use locally built libcheck + set(CHECK_ROOT_DIR "${LIBCHECK_PREFIX}") + find_package(CHECK) + endif(LIBCHECK_RETCODE) + endif(NOT CHECK_WORKING OR FORCE_CHECK_BUILD) + + # use fluxcapacitor to mock time + if(OS_PLATFORM STREQUAL "OS_LINUX") + set(FLUXCAP_PREFIX "${CMAKE_BINARY_DIR}/fluxcapacitor") + execute_process( + COMMAND "bash" "${PROJECT_SOURCE_DIR}/ci/install-fluxcapacitor.sh" "${FLUXCAP_PREFIX}" + TIMEOUT 60 # if this doesn't build in 60 seconds something is hosed + RESULT_VARIABLE FLUXCAP_RETCODE + ) + if(FLUXCAP_RETCODE) # non-zero means error + message(STATUS "build fluxcapacitor failed, return code: " ${FLUXCAP_RETCODE}) + else(FLUXCAP_RETCODE) + set(FLUXCAP_BINARY "${FLUXCAP_PREFIX}/fluxcapacitor") + message(STATUS "fluxcapacitor available at: " ${FLUXCAP_BINARY}) + endif(FLUXCAP_RETCODE) + endif(OS_PLATFORM STREQUAL "OS_LINUX") +endif(HAVE_TEST) if (HAVE_ITT_INSTRUMENTATION) if(PKG_CONFIG_FOUND) @@ -177,6 +174,12 @@ endif(HAVE_ITT_INSTRUMENTATION) find_package(Threads) +if (HAVE_COVERAGE) + if(NOT ${CMAKE_BUILD_TYPE} MATCHES Debug) + message(WARNING "Code coverage results with an optimised (non-Debug) build may be misleading" ) + endif() + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage") +endif(HAVE_COVERAGE) # where to find include files include_directories( @@ -190,15 +193,14 @@ include_directories( add_subdirectory(src) -if(CHECK_FOUND) +if(HAVE_TEST) include_directories(${include_directories} ${CHECK_INCLUDES}) add_subdirectory(test) -endif(CHECK_FOUND) +endif(HAVE_TEST) if(HAVE_RUST) include(CMakeCargo) add_subdirectory(rust) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DHAVE_RUST=1") if (${CMAKE_VERSION} VERSION_LESS "3.13.0") # CMakeCargo requires the use of some newer features of cmake @@ -217,15 +219,43 @@ endif() # print a summary # ################### +message(STATUS "<<++=====------------------\\/------------------=====++>>") +message(STATUS "<<++ ccommon summary ++>>") +message(STATUS "<<++=====------------------/\\------------------=====++>>") +message(STATUS "=============CMake related=============") message(STATUS "CMAKE_BUILD_TYPE: " ${CMAKE_BUILD_TYPE}) - message(STATUS "PLATFORM: " ${OS_PLATFORM}) - message(STATUS "CPPFLAGS: " ${CMAKE_CPP_FLAGS}) message(STATUS "CFLAGS: " ${CMAKE_C_FLAGS}) +message(STATUS "=======================================") +message(STATUS "=======Status of system features=======") message(STATUS "HAVE_SIGNAME: " ${HAVE_SIGNAME}) - message(STATUS "HAVE_BACKTRACE: " ${HAVE_BACKTRACE}) - -message(STATUS "CHECK_FOUND: " ${CHECK_FOUND}) +message(STATUS "HAVE_ACCEPT4: " ${HAVE_ACCEPT4}) +if(OS_PLATFORM STREQUAL "OS_LINUX") + message(STATUS "HAVE_TIME64: " ${HAVE_TIME64}) +endif() +message(STATUS "=======================================") + +message(STATUS "======Status of optional features======") +message(STATUS "HAVE_RUST: " ${HAVE_RUST}) +message(STATUS "HAVE_ASSERT_LOG: " ${HAVE_ASSERT_LOG}) +message(STATUS "HAVE_ASSERT_PANIC: " ${HAVE_ASSERT_PANIC}) +message(STATUS "HAVE_LOGGING: " ${HAVE_LOGGING}) +message(STATUS "HAVE_STATS: " ${HAVE_STATS}) +message(STATUS "HAVE_ITT_INSTRUMENTATION: " ${HAVE_ITT_INSTRUMENTATION}) +message(STATUS "HAVE_DEBUG_MM: " ${HAVE_DEBUG_MM}) +message(STATUS "HAVE_TEST: " ${HAVE_TEST}) +message(STATUS "HAVE_COVERAGE: " ${HAVE_COVERAGE}) +message(STATUS "=======================================") + + +if(DUMP_ALL) + message(STATUS "<<++=====------------------\\/------------------=====++>>") + get_cmake_property(_variableNames VARIABLES) + foreach (_variableName ${_variableNames}) + message(STATUS "${_variableName}=${${_variableName}}") + endforeach() + message(STATUS "<<++=====------------------/\\------------------=====++>>") +endif() diff --git a/deps/ccommon/ci/cargo.sh b/deps/ccommon/ci/cargo.sh index a24b2dcfa..8a8c25725 100755 --- a/deps/ccommon/ci/cargo.sh +++ b/deps/ccommon/ci/cargo.sh @@ -20,7 +20,8 @@ cat > Cargo.toml <
> Cargo.toml done diff --git a/deps/ccommon/ci/install-check.sh b/deps/ccommon/ci/install-check.sh index 0658135c4..070c4ebea 100755 --- a/deps/ccommon/ci/install-check.sh +++ b/deps/ccommon/ci/install-check.sh @@ -17,14 +17,17 @@ TEMP="$(mktemp -d -t TEMP.XXXXXXX)" || die "failed to make tmpdir" cleanup() { [[ -n "${TEMP:-}" ]] && rm -rf "${TEMP}"; } trap cleanup EXIT -realpath() { python -c "from __future__ import print_function; import os,sys; print(os.path.realpath(sys.argv[1]))" "$1"; } +realpath() { python3 -c "import os,sys; print(os.path.realpath(sys.argv[1]))" "$1"; } TOPLEVEL="$(cd "$(dirname "$(realpath "$0" >/dev/null || exit 1)")" && git rev-parse --show-toplevel)" || die 'failed to find TOPLEVEL' -CHECK_VERSION=0.12.0 +CHECK_VERSION="0.14.0" CHECK_TARBALL="check-${CHECK_VERSION}.tar.gz" CHECK_DIR="check-${CHECK_VERSION}" +CHECK_LOG="build-check.log" + +echo "building and installing check" >&2 echo "building and installing check" >&2 @@ -33,10 +36,12 @@ echo "building and installing check" >&2 wget "https://github.com/libcheck/check/releases/download/${CHECK_VERSION}/${CHECK_TARBALL}" && tar xfz "${CHECK_TARBALL}" && cd "${CHECK_DIR}" && - ./configure --prefix="$CHECK_PREFIX" && - make && + mkdir build && + cd build && + cmake -DCMAKE_INSTALL_PREFIX="${CHECK_PREFIX}" .. && + make -j && make install -) >$TEMP/cmake-build.log 2>&1 +) >$TEMP/${CHECK_LOG} 2>&1 RESULT=$? if [[ $RESULT -ne 0 ]]; then @@ -45,7 +50,7 @@ check build failed! log below: EOS - cat $TEMP/cmake-build.log + cat $TEMP/${CHECK_LOG} else echo "Success!" >&2 fi diff --git a/deps/ccommon/ci/install-fluxcapacitor.sh b/deps/ccommon/ci/install-fluxcapacitor.sh new file mode 100755 index 000000000..643350f63 --- /dev/null +++ b/deps/ccommon/ci/install-fluxcapacitor.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +set -euo pipefail +IFS=$'\n\t' + +die() { echo "fatal: $*" >&2; exit 1; } + +if [[ $# -lt 1 ]]; then + echo "Usage: $0 fluxcap-install-path" + exit 1 +fi + +FLUXCAP_PREFIX="$1" +shift + +TEMP="$(mktemp -d -t TEMP.XXXXXXX)" || die "failed to make tmpdir" +cleanup() { [[ -n "${TEMP:-}" ]] && rm -rf "${TEMP}"; } +trap cleanup EXIT + +realpath() { python3 -c "import os,sys; print(os.path.realpath(sys.argv[1]))" "$1"; } + +TOPLEVEL="$(cd "$(dirname "$(realpath "$0" >/dev/null || exit 1)")" && git rev-parse --show-toplevel)" || die 'failed to find TOPLEVEL' + + +FLUXCAP_VERSION=0.1 +FLUXCAP_TARBALL="${FLUXCAP_VERSION}.tar.gz" +FLUXCAP_DIR="fluxcapacitor-${FLUXCAP_VERSION}" +FLUXCAP_LOG="build-fluxcapacitor.log" + +echo "building fluxcapacitor" >&2 + +( + cd "$TEMP" && + wget "https://github.com/thinkingfish/fluxcapacitor/archive/${FLUXCAP_TARBALL}" && + tar xfz "${FLUXCAP_TARBALL}" && + pwd && + ls . && + cd "${FLUXCAP_DIR}" && + make build +) >$TEMP/${FLUXCAP_LOG} 2>&1 + +RESULT=$? +if [[ $RESULT -ne 0 ]]; then + cat >&2 <&2 +fi + +exit $RESULT diff --git a/deps/ccommon/ci/run.sh b/deps/ccommon/ci/run.sh index d6f54cb19..6e99dc3c3 100755 --- a/deps/ccommon/ci/run.sh +++ b/deps/ccommon/ci/run.sh @@ -9,7 +9,6 @@ export PATH=$HOME/.cargo/bin:$PATH cmake_cmd=( cmake - -DBUILD_AND_INSTALL_CHECK=yes ) if [[ -n "${RUST_ENABLED:-}" ]]; then diff --git a/deps/ccommon/cmake/FindCheck.cmake b/deps/ccommon/cmake/FindCHECK.cmake similarity index 100% rename from deps/ccommon/cmake/FindCheck.cmake rename to deps/ccommon/cmake/FindCHECK.cmake diff --git a/deps/ccommon/test/channel/pipe/check_pipe.c b/deps/ccommon/test/channel/pipe/check_pipe.c index 5072f6132..5192119e2 100644 --- a/deps/ccommon/test/channel/pipe/check_pipe.c +++ b/deps/ccommon/test/channel/pipe/check_pipe.c @@ -104,7 +104,8 @@ START_TEST(test_read_blocking) pthread_t thread; #define READ_MESSAGE_LENGTH 12 #define SLEEP_TIME 500000 -#define TOLERANCE_TIME 100000 +#define TOLERANCE_UNDER 10000 +#define TOLERANCE_OVER 100000 char read_message[READ_MESSAGE_LENGTH]; test_reset(); duration_reset(&duration); @@ -126,8 +127,8 @@ START_TEST(test_read_blocking) duration_stop(&duration); pthread_join(thread, NULL); - ck_assert_int_ge(duration_us(&duration), SLEEP_TIME); - ck_assert_int_le(duration_us(&duration), SLEEP_TIME + TOLERANCE_TIME); + ck_assert_int_ge(duration_us(&duration), SLEEP_TIME - TOLERANCE_UNDER); + ck_assert_int_le(duration_us(&duration), SLEEP_TIME + TOLERANCE_OVER); ck_assert_str_eq(write_message, read_message); diff --git a/deps/ccommon/test/time/wheel/check_wheel.c b/deps/ccommon/test/time/wheel/check_wheel.c index 4bd4df0ee..177eb0008 100644 --- a/deps/ccommon/test/time/wheel/check_wheel.c +++ b/deps/ccommon/test/time/wheel/check_wheel.c @@ -116,13 +116,14 @@ END_TEST START_TEST(test_timing_wheel_recur) { -#define TICK_NS 50000000 +#define TICK_NS 500000000 +#define SLEEP_NS 450000000 #define NSLOT 3 #define NTICK 2 struct timeout tick, delay; struct timing_wheel *tw; - struct timespec ts = (struct timespec){0, TICK_NS}; + struct timespec ts = (struct timespec){0, SLEEP_NS}; int i = 0; test_reset(); @@ -134,21 +135,32 @@ START_TEST(test_timing_wheel_recur) timing_wheel_start(tw); ck_assert_int_le(timeout_ns(&tw->due), TICK_NS); + /* recurring events are inserted into the next tick at the earliest */ timing_wheel_insert(tw, &delay, true, _incr_cb, &i); - /* tick unchanged */ + /* tick not advanced yet, no event processed */ timing_wheel_execute(tw); ck_assert_int_eq(tw->nprocess, 0); ck_assert_int_eq(tw->nevent, 1); - /* next 2 tick */ nanosleep(&ts, NULL); nanosleep(&ts, NULL); + nanosleep(&ts, NULL); + /* elapsed time between 2 and 3 ticks. + * The first tick started empty; the next ticket has one + * event, and we should process it, and inserting it into + * the tick after that (because delay < tick, and re-insert + * will move the event by at least one slot). + * Despite the re-inserted event being due by wall clock + * since we process the tick at the end of that tick, we + * should only see one event processed. + */ timing_wheel_execute(tw); ck_assert_int_eq(tw->nevent, 1); ck_assert_int_eq(tw->nprocess, 1); ck_assert_int_eq(i, 1); nanosleep(&ts, NULL); + /* elapsed time now between 3 and 4 ticks */ timing_wheel_execute(tw); ck_assert_int_eq(tw->nevent, 1); ck_assert_int_eq(tw->nprocess, 2); @@ -163,6 +175,7 @@ START_TEST(test_timing_wheel_recur) #undef NTICK #undef NSLOT +#undef SLEEP_NS #undef TICK_NS } END_TEST diff --git a/src/rust-util/pelikan-sys/src/storage/slab.rs b/src/rust-util/pelikan-sys/src/storage/slab.rs index fca24aee3..5d52c7fbd 100644 --- a/src/rust-util/pelikan-sys/src/storage/slab.rs +++ b/src/rust-util/pelikan-sys/src/storage/slab.rs @@ -71,7 +71,11 @@ unsafe impl ccommon::metric::Metrics for slab_metrics_st { ACTION( item_link, METRIC_COUNTER, "# items inserted to HT" ), ACTION( item_unlink, METRIC_COUNTER, "# items removed from HT" ), ACTION( item_keyval_byte, METRIC_GAUGE, "key+val in bytes, linked" ), - ACTION( item_val_byte, METRIC_GAUGE, "value only in bytes" ) + ACTION( item_val_byte, METRIC_GAUGE, "value only in bytes" ), + ACTION( hash_lookup, METRIC_COUNTER, "# of hash lookups" ), + ACTION( hash_insert, METRIC_COUNTER, "# of hash inserts" ), + ACTION( hash_remove, METRIC_COUNTER, "# of hash deletes" ), + ACTION( hash_traverse, METRIC_COUNTER, "# of nodes touched" ) } } } diff --git a/src/storage/slab/hashtable.c b/src/storage/slab/hashtable.c index 9512e5b9c..2d29a35c1 100644 --- a/src/storage/slab/hashtable.c +++ b/src/storage/slab/hashtable.c @@ -1,4 +1,4 @@ -#include "hashtable.h" +#include "slab.h" #include #include @@ -57,11 +57,14 @@ hashtable_create(uint32_t hash_power) } void -hashtable_destroy(struct hash_table *ht) +hashtable_destroy(struct hash_table **ht_p) { + struct hash_table *ht = *ht_p; if (ht != NULL && ht->table != NULL) { cc_free(ht->table); } + + *ht_p = NULL; } static struct item_slh * @@ -85,6 +88,7 @@ hashtable_put(struct item *it, struct hash_table *ht) SLIST_INSERT_HEAD(bucket, it, i_sle); ++(ht->nhash_item); + INCR(slab_metrics, hash_insert); } void @@ -98,6 +102,8 @@ hashtable_delete(const char *key, uint32_t klen, struct hash_table *ht) bucket = _get_bucket(key, klen, ht); for (prev = NULL, it = SLIST_FIRST(bucket); it != NULL; prev = it, it = SLIST_NEXT(it, i_sle)) { + INCR(slab_metrics, hash_traverse); + /* iterate through bucket to find item to be removed */ if ((klen == it->klen) && cc_memcmp(key, item_key(it), klen) == 0) { /* found item */ @@ -112,6 +118,7 @@ hashtable_delete(const char *key, uint32_t klen, struct hash_table *ht) } --(ht->nhash_item); + INCR(slab_metrics, hash_remove); } struct item * @@ -123,9 +130,13 @@ hashtable_get(const char *key, uint32_t klen, struct hash_table *ht) ASSERT(key != NULL); ASSERT(klen != 0); + INCR(slab_metrics, hash_lookup); + bucket = _get_bucket(key, klen, ht); /* iterate through bucket looking for item */ for (it = SLIST_FIRST(bucket); it != NULL; it = SLIST_NEXT(it, i_sle)) { + INCR(slab_metrics, hash_traverse); + if ((klen == it->klen) && cc_memcmp(key, item_key(it), klen) == 0) { /* found item */ return it; @@ -134,3 +145,43 @@ hashtable_get(const char *key, uint32_t klen, struct hash_table *ht) return NULL; } + +/* + * Expand the hashtable to the next power of 2. + * This is an expensive operation and should _not_ be used in production or + * during latency-related tests. It is included mostly for simulation around + * the storage component. + */ +struct hash_table * +hashtable_double(struct hash_table *ht) +{ + struct hash_table *new_ht; + uint32_t new_hash_power; + uint64_t new_size; + + new_hash_power = ht->hash_power + 1; + new_size = HASHSIZE(new_hash_power); + + new_ht = hashtable_create(new_size); + if (new_ht == NULL) { + return ht; + } + + /* copy to new hash table */ + for (uint32_t i = 0; i < HASHSIZE(ht->hash_power); ++i) { + struct item *it, *next; + struct item_slh *bucket, *new_bucket; + + bucket = &ht->table[i]; + SLIST_FOREACH_SAFE(it, bucket, i_sle, next) { + new_bucket = _get_bucket(item_key(it), it->klen, new_ht); + SLIST_REMOVE(bucket, it, item, i_sle); + SLIST_INSERT_HEAD(new_bucket, it, i_sle); + } + } + + hashtable_destroy(&ht); + + return new_ht; +} + diff --git a/src/storage/slab/hashtable.h b/src/storage/slab/hashtable.h index ba2a370d4..994449cfe 100644 --- a/src/storage/slab/hashtable.h +++ b/src/storage/slab/hashtable.h @@ -12,8 +12,11 @@ struct hash_table { #define HASHMASK(_n) (HASHSIZE(_n) - 1) struct hash_table *hashtable_create(uint32_t hash_power); -void hashtable_destroy(struct hash_table *ht); +void hashtable_destroy(struct hash_table **ht_p); void hashtable_put(struct item *it, struct hash_table *ht); void hashtable_delete(const char *key, uint32_t klen, struct hash_table *ht); struct item *hashtable_get(const char *key, uint32_t klen, struct hash_table *ht); + + +struct hash_table *hashtable_double(struct hash_table *ht); /* best effort expansion */ diff --git a/src/storage/slab/slab.c b/src/storage/slab/slab.c index 7839ee67d..b79b53e0c 100644 --- a/src/storage/slab/slab.c +++ b/src/storage/slab/slab.c @@ -553,7 +553,7 @@ slab_teardown(void) log_warn("%s has never been set up", SLAB_MODULE_NAME); } - hashtable_destroy(hash_table); + hashtable_destroy(&hash_table); _slab_heapinfo_teardown(); _slab_slabclass_teardown(); slab_metrics = NULL; diff --git a/src/storage/slab/slab.h b/src/storage/slab/slab.h index 908a0b5ac..82c27178e 100644 --- a/src/storage/slab/slab.h +++ b/src/storage/slab/slab.h @@ -79,7 +79,11 @@ typedef struct { ACTION( item_link, METRIC_COUNTER, "# items inserted to HT" )\ ACTION( item_unlink, METRIC_COUNTER, "# items removed from HT" )\ ACTION( item_keyval_byte, METRIC_GAUGE, "key+val in bytes, linked" )\ - ACTION( item_val_byte, METRIC_GAUGE, "value only in bytes" ) + ACTION( item_val_byte, METRIC_GAUGE, "value only in bytes" )\ + ACTION( hash_lookup, METRIC_COUNTER, "# of hash lookups" )\ + ACTION( hash_insert, METRIC_COUNTER, "# of hash inserts" )\ + ACTION( hash_remove, METRIC_COUNTER, "# of hash deletes" )\ + ACTION( hash_traverse, METRIC_COUNTER, "# of nodes touched" ) typedef struct { SLAB_METRIC(METRIC_DECLARE)