From f02419eb5c10810349309898f0f9255e35bd9d0e Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Thu, 28 Mar 2024 18:28:36 +0200 Subject: [PATCH 001/105] Fix macro redefinition warnings, add bmqstoragetool.td target Signed-off-by: Aleksandr Ivanov --- etc/cmake/BMQTest.cmake | 17 +++++++---------- .../bmqstoragetool/m_bmqstoragetool_testutils.h | 9 +++++++++ 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/etc/cmake/BMQTest.cmake b/etc/cmake/BMQTest.cmake index 2489985720..6b5468bd79 100644 --- a/etc/cmake/BMQTest.cmake +++ b/etc/cmake/BMQTest.cmake @@ -194,6 +194,13 @@ function(bmq_add_application_test target) add_custom_target(${target}.t) endif() add_dependencies(${target}.t ${lib_target}.t) + + if(_COMPAT) + if (NOT TARGET ${target}.td) + add_custom_target(${target}.td) + endif() + add_dependencies(${target}.td ${lib_target}.t) + endif() endif() if (${lib_target}_TEST_TARGETS) @@ -203,16 +210,6 @@ function(bmq_add_application_test target) set(td_manifest) if(${lib_target}_TEST_TARGETS) - - if(NOT TARGET ${lib_target}.t) - add_custom_target(${lib_target}.t) - if(_COMPAT) - add_custom_target(${lib_target}.td) - endif() - endif() - - add_dependencies(${lib_target}.t ${${lib_target}_TEST_TARGETS}) - if(_COMPAT) foreach(test_target ${${lib_target}_TEST_TARGETS}) string(REPLACE ".t" "" component ${test_target}) diff --git a/src/applications/bmqstoragetool/m_bmqstoragetool_testutils.h b/src/applications/bmqstoragetool/m_bmqstoragetool_testutils.h index f8bf024d88..89e9ba1626 100644 --- a/src/applications/bmqstoragetool/m_bmqstoragetool_testutils.h +++ b/src/applications/bmqstoragetool/m_bmqstoragetool_testutils.h @@ -51,6 +51,15 @@ // GMOCK #include +// Undefine macroses from gtest.h which are defined in mwctst_testhelper.h +#undef ASSERT_EQ +#undef ASSERT_NE +#undef ASSERT_LT +#undef ASSERT_LE +#undef ASSERT_GT +#undef ASSERT_GE +#undef TEST_F +#undef TEST // TEST DRIVER #include From 9117ac76a92f12e4cd39addd32649977b1415490 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Thu, 6 Jun 2024 15:22:12 +0300 Subject: [PATCH 002/105] Fix order of objects creation in unit test Signed-off-by: Aleksandr Ivanov --- src/applications/bmqtool/m_bmqtool_inpututil.t.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/applications/bmqtool/m_bmqtool_inpututil.t.cpp b/src/applications/bmqtool/m_bmqtool_inpututil.t.cpp index 5c5b936213..60f1d81b68 100644 --- a/src/applications/bmqtool/m_bmqtool_inpututil.t.cpp +++ b/src/applications/bmqtool/m_bmqtool_inpututil.t.cpp @@ -290,9 +290,9 @@ static void test2_loadMessageFromFileTest() // Check payload ASSERT_EQ_D(test.d_line, payload.str(), test.d_expectedPayload); // Check properties (deserialize into properties instance) - bmqa::MessageProperties messageProperties(s_allocator_p); bdlbb::PooledBlobBufferFactory bufferFactory(128, s_allocator_p); bdlbb::Blob blob(&bufferFactory, s_allocator_p); + bmqa::MessageProperties messageProperties(s_allocator_p); bdlbb::BlobUtil::append(&blob, properties.str().c_str(), static_cast(properties.str().size())); From 950cbf9f2b60fa1509e2dce250ca8aec96fd6af3 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Fri, 5 Jul 2024 18:06:39 +0300 Subject: [PATCH 003/105] Add sanitizers folder with build scripts Signed-off-by: Aleksandr Ivanov --- .../workflows/sanitizers/BBToolchain64.cmake | 148 ++++++++++ .github/workflows/sanitizers/Dockerfile | 6 + .github/workflows/sanitizers/build_deps.sh | 84 ++++++ .../sanitizers/clang-libcxx-asan.cmake | 113 ++++++++ .../sanitizers/clang-libcxx-msan.cmake | 121 ++++++++ .github/workflows/sanitizers/sanitizers.json | 48 ++++ .github/workflows/sanitizers/script.sh | 267 ++++++++++++++++++ 7 files changed, 787 insertions(+) create mode 100644 .github/workflows/sanitizers/BBToolchain64.cmake create mode 100644 .github/workflows/sanitizers/Dockerfile create mode 100644 .github/workflows/sanitizers/build_deps.sh create mode 100644 .github/workflows/sanitizers/clang-libcxx-asan.cmake create mode 100644 .github/workflows/sanitizers/clang-libcxx-msan.cmake create mode 100644 .github/workflows/sanitizers/sanitizers.json create mode 100644 .github/workflows/sanitizers/script.sh diff --git a/.github/workflows/sanitizers/BBToolchain64.cmake b/.github/workflows/sanitizers/BBToolchain64.cmake new file mode 100644 index 0000000000..7a35b019e3 --- /dev/null +++ b/.github/workflows/sanitizers/BBToolchain64.cmake @@ -0,0 +1,148 @@ +# BBToolchain64.cmake *GENERATED FILE - DO NOT EDIT* -*-cmake-*- + +cmake_minimum_required (VERSION 3.22) + +#@PURPOSE: Configure compiler, ABI flags and directory structure. +# +#@DESCRIPTION: This is a cmake toolchain file, designed to be specified by +# setting the CMAKE_TOOLCHAIN_FILE variable either as a command line parameter +# by using -D or by calling "set" before the top-level CMakeLists.txt call to +# project(). **This file should NOT be included directly**. The purpose +# of the toolchain file is to prepare the build directory for cross-compiling. +# In our context, this means setting the compiler to be the production ready +# compiler version as well as the flags necessary for ABI compatibility so that +# the artifacts can be used on any machine in production. Search paths for +# CMake find_### methods as well as the CMake PkgConfig integration are +# configured here. In addition, this file will set basic variables necessary +# for navigating both the directory structure. These variables should only +# be referenced by distribution modules such as Pekludge.cmake, as they are not +# guaranteed to exist in alternative toolchains. Extending modules may take +# use of the following variables: +# * DISTRIBUTION_REFROOT - Root of dpkg artifacts. +# * ROBOLIB_REFROOT - Root of classic robolib artifacts. +# * IS_64BIT - Boolean value needed for directory navigation. +# +# Note that both refroot variables will first use the cmake command line +# parameter definition, then the environment variable if set and finally will +# use the toolchain file location to infer the refroots. + +# Set base environment variables. +# if(NOT DEFINED DISTRIBUTION_REFROOT) +# if(DEFINED ENV{DISTRIBUTION_REFROOT}) +# set(DISTRIBUTION_REFROOT "$ENV{DISTRIBUTION_REFROOT}/" CACHE STRING "BB Dpkg root set from environment variable.") +# else() +# get_filename_component(REFROOT ${CMAKE_CURRENT_LIST_DIR}/../../../../ REALPATH) +# set(DISTRIBUTION_REFROOT ${REFROOT}/ CACHE STRING "BB Dpkg root set from toolchain file location.") +# endif() +# endif() +# if(NOT DEFINED ROBOLIB_REFROOT) +# if(DEFINED ENV{ROBOLIB_REFROOT}) +# set(ROBOLIB_REFROOT "$ENV{ROBOLIB_REFROOT}/" CACHE STRING "BB Robo root set from environment variable.") +# else() +# get_filename_component(REFROOT ${CMAKE_CURRENT_LIST_DIR}/../../../../ REALPATH) +# set(ROBOLIB_REFROOT ${REFROOT}/ CACHE STRING "BB Robo root set from toolchain file location.") +# endif() +# endif() +set(IS_64BIT yes CACHE BOOL "Bloomberg tool chain bitness.") + +# Set distribution compilers and ABI flags. +# set(CMAKE_C_COMPILER ${DISTRIBUTION_REFROOT}/opt/bb/bin/gcc CACHE FILEPATH "C Compiler path") +# set(CMAKE_CXX_COMPILER ${DISTRIBUTION_REFROOT}/opt/bb/bin/g++ CACHE FILEPATH "C++ Compiler path") +# set(CMAKE_Fortran_COMPILER ${DISTRIBUTION_REFROOT}/opt/bb/bin/gfortran CACHE FILEPATH "Fortran Compiler path") + +# Set the compiler flags. Use the CACHE instead of the *_INIT variants +# which are modified in the in the compiler initialization modules. +set(CMAKE_C_FLAGS "-march=westmere -m64 -fno-strict-aliasing" CACHE STRING "Bloomberg ABI C flags.") +set(CMAKE_C_FLAGS_RELWITHDEBINFO "-g -O2 -fno-omit-frame-pointer" CACHE STRING "Bloomberg ABI C flags.") +set(CMAKE_CXX_STANDARD 20 CACHE STRING "Bloomberg C++ standard.") +set(CMAKE_CXX_STANDARD_REQUIRED 1 CACHE BOOL "Bloomberg standard version is required.") +set(CMAKE_CXX_FLAGS "-D_GLIBCXX_USE_CXX11_ABI=0 -march=westmere -m64 -fno-strict-aliasing" CACHE STRING "Bloomberg ABI C++ flags.") +set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-g -O2 -fno-omit-frame-pointer" CACHE STRING "Bloomberg ABI C++ flags.") +set(CMAKE_Fortran_FLAGS "-march=westmere -mstackrealign -fasynchronous-unwind-tables -m64 -std=legacy -Wall -Wno-overwrite-recursive -fimplicit-none -fdec -frecursive -fno-automatic -fno-range-check -fno-align-commons -fd-lines-as-comments -static-libgfortran -fno-dec-promotion -fno-dec-old-init -fno-dec-non-integer-index -fno-dec-sequence -fno-dec-add-missing-indexes -fno-dec-non-logical-if -fno-dec-override-kind -fallow-invalid-boz -fallow-argument-mismatch" CACHE STRING "Bloomberg ABI Fortran flags.") +set(CMAKE_Fortran_FLAGS_RELWITHDEBINFO "-g -O2 -fno-omit-frame-pointer" CACHE STRING "Bloomberg ABI Fortran flags.") + +# Compile variable overrides. +set(CMAKE_USER_MAKE_RULES_OVERRIDE ${CMAKE_CURRENT_LIST_FILE}) # Sets this file as the override for the builtin compiler variables and rules +set(CMAKE_C_LINK_FLAGS "-fasynchronous-unwind-tables -Wl,--enable-new-dtags -Wl,--gc-sections -Wl,-uplink_timestamp___") +set(CMAKE_CXX_LINK_FLAGS "-fasynchronous-unwind-tables -Wl,--enable-new-dtags -Wl,--gc-sections -Wl,-uplink_timestamp___") +set(CMAKE_Fortran_LINK_FLAGS "") + +# Ninja generator can preprocess source files on some platforms to build a better dependency tree than the Makefiles generator. +# We don't use include directives in our Fortran here at Bloomberg so this is not needed. +set(CMAKE_Fortran_PREPROCESS OFF) + +set(CMAKE_Fortran_IMPLICIT_LINK_LIBRARIES "-l:libgfortran.a;-l:libquadmath.a") + +# Set the platform RPATH for production deployment. +set(CMAKE_PLATFORM_REQUIRED_RUNTIME_PATH /usr/lib64 /lib64 /bb/bin/so/64/NS02 /bb/bin/so/64 /bb/bin .) + +# Set the local module path to include standard Bloomberg modules. +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR} ${DISTRIBUTION_REFROOT}/opt/bb/share/cmake/Modules) + +# Set the program path for pkg-config. Versions 0.29.1 and older without our +# patch must be avoided since they do not scale to the size of our distribution. +# find_program(PKG_CONFIG_EXECUTABLE pkg-config PATHS +# ${DISTRIBUTION_REFROOT}/opt/bb/lib64/bin +# /opt/bb/lib64/bin +# NO_SYSTEM_ENVIRONMENT_PATH) + +# Toolchain files could be invoked multiple times in a single CMake run, +# therefore we need to check if we have appended the refroot pkgconfig path +# in a previous invokation before appending. We cannot use a cached variable +# to achieve this because the cache is not available between separate +# invokations of the toolchain file. +# set(ROBO_PKG_CONFIG_PATH "${DISTRIBUTION_REFROOT}/opt/bb/lib64/robo/pkgconfig:${DISTRIBUTION_REFROOT}/opt/bb/lib64/pkgconfig" CACHE STRING "The location of the robo pkgconfig files.") +# if (DEFINED ENV{PKG_CONFIG_PATH} AND NOT "$ENV{PKG_CONFIG_PATH}" MATCHES ".*${ROBO_PKG_CONFIG_PATH}$") +# message(STATUS "WARNING: Using user supplied PKG_CONFIG_PATH=$ENV{PKG_CONFIG_PATH}") +# endif() +# if (NOT "$ENV{PKG_CONFIG_PATH}" MATCHES ".*${ROBO_PKG_CONFIG_PATH}$") +# set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:${ROBO_PKG_CONFIG_PATH}") +# endif() + +# set(ENV{PKG_CONFIG_SYSROOT_DIR} ${DISTRIBUTION_REFROOT}) + +# Set the path for looking up includes, libs and files. +# list(APPEND CMAKE_SYSTEM_PREFIX_PATH ${DISTRIBUTION_REFROOT}/opt/bb) +set_property(GLOBAL PROPERTY FIND_LIBRARY_USE_LIB64_PATHS yes) + +# Set the installation directory for libraries. This is for use as the +# DESTINATION for calls to the cmake 'install' command. Note that this is not +# an actual cmake variable, but has become a de facto standard for use in +# systems that employ a lib/lib64 deployment architecture. To allow for use +# without a toolchain, use "./${CMAKE_INSTALL_LIBDIR}" to avoid configuration +# errors when this variable is not set. +set(CMAKE_INSTALL_LIBDIR lib64) + +# if ("amd64" STREQUAL "aix6-powerpc" AND IS_64BIT) +# # On AIX, the archiver defaults to 32 bit only. This is a known issue +# # in CMake and unlikely to be fixed. The following is the recommended +# # workaround. Note that the AIX archiver can also use an environment +# # variable, but in cmake, there is no way to set the env at build time. +# set(CMAKE_CXX_ARCHIVE_CREATE +# " -X64 cr ") +# set(CMAKE_CXX_ARCHIVE_APPEND +# " -X64 r ") +# set(CMAKE_CXX_ARCHIVE_FINISH +# " -X64 ") +# set(CMAKE_C_ARCHIVE_CREATE +# " -X64 cr ") +# set(CMAKE_C_ARCHIVE_APPEND +# " -X64 r ") +# set(CMAKE_C_ARCHIVE_FINISH +# " -X64 ") +# set(CMAKE_Fortran_ARCHIVE_CREATE +# " -X64 cr ") +# set(CMAKE_Fortran_ARCHIVE_APPEND +# " -X64 r ") +# set(CMAKE_Fortran_ARCHIVE_FINISH +# " -X64 ") +# endif() + +# ----------------------------------------------------------------------------- +# NOTICE: +# Copyright (C) Bloomberg L.P., 2016 +# All Rights Reserved. +# Property of Bloomberg L.P. (BLP) +# This software is made available solely pursuant to the +# terms of a BLP license agreement which governs its use. +# ----------------------------- END-OF-FILE ----------------------------------- diff --git a/.github/workflows/sanitizers/Dockerfile b/.github/workflows/sanitizers/Dockerfile new file mode 100644 index 0000000000..0d11d6012f --- /dev/null +++ b/.github/workflows/sanitizers/Dockerfile @@ -0,0 +1,6 @@ +FROM ubuntu:jammy-20240627.1 + +#RUN apt update \ +# && apt install -y build-essential cmake clang-format clang-tidy lcov libbenchmark-dev libclang-dev libgoogle-perftools-dev python3 python3-pip python-is-python3 \ +# && rm -rf /var/lib/apt/lists/* /var/log/dpkg.log + diff --git a/.github/workflows/sanitizers/build_deps.sh b/.github/workflows/sanitizers/build_deps.sh new file mode 100644 index 0000000000..b256d9d288 --- /dev/null +++ b/.github/workflows/sanitizers/build_deps.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +# This script downloads, builds, and installs the required build dependencies of BMQ +# from github.com/bloomberg. Software packages are installed to the /opt/bb prefix. + +set -euxo pipefail + +if [ $# == 1 ]; then + if [[ $1 == "only-download" ]]; then + DO_BUILD=false + else + echo "Unexpected optional argument, only 'only-download' is supported" + exit 1 + fi +else + DO_BUILD=true +fi + +fetch_git() { + local org=$1 + local repo=$2 + mkdir -p srcs + + if [ -d "srcs/${repo}" ]; then + return 0 + fi + + if [ -z "${3:-}" ] + then + # Clone the latest 'main' branch if no specific release tag provided + local branch="main" + curl -SL "https://github.com/${org}/${repo}/archive/refs/heads/${branch}.tar.gz" | tar -xzC srcs/ + mv "srcs/${repo}-${branch}" "srcs/${repo}" + else + local tag=$3 + curl -SL "https://github.com/${org}/${repo}/archive/refs/tags/${tag}.tar.gz" | tar -xzC srcs/ + mv "srcs/${repo}-${tag}" "srcs/${repo}" + fi +} + +fetch_deps() { + fetch_git bloomberg bde-tools 4.8.0.0 + fetch_git bloomberg bde 4.8.0.0 + fetch_git bloomberg ntf-core 2.4.2 +} + +configure() { + PATH="$PATH:$(realpath srcs/bde-tools/bin)" + export PATH + eval "$(bbs_build_env -u opt_64_cpp17)" +} + +build_bde() { + pushd srcs/bde + bbs_build configure + bbs_build build -j8 + bbs_build --install=/opt/bb --prefix=/ install + popd +} + +build_ntf() { + pushd srcs/ntf-core + ./configure \ + --keep \ + --prefix /opt/bb \ + --without-usage-examples \ + --without-applications \ + --without-warnings-as-errors \ + --ufid opt_64_cpp17 + make -j8 + make install + popd +} + +build() { + build_bde + build_ntf +} + +fetch_deps +configure +if [ "${DO_BUILD}" = true ]; then + build +fi diff --git a/.github/workflows/sanitizers/clang-libcxx-asan.cmake b/.github/workflows/sanitizers/clang-libcxx-asan.cmake new file mode 100644 index 0000000000..1230eba33e --- /dev/null +++ b/.github/workflows/sanitizers/clang-libcxx-asan.cmake @@ -0,0 +1,113 @@ +# Compiler-less toolchain for building with {Clang, libc++} + AddressSanitizer. +# The actual compiler is passed via `CXX` and `CC` environment variables. +#-=- + +cmake_minimum_required (VERSION 3.25) + +# if(NOT DEFINED DISTRIBUTION_REFROOT) +# if(DEFINED ENV{DISTRIBUTION_REFROOT}) +# set(DISTRIBUTION_REFROOT "$ENV{DISTRIBUTION_REFROOT}/" CACHE STRING "BB Dpkg root set from environment variable.") +# else() +# get_filename_component(REFROOT ${CMAKE_CURRENT_LIST_DIR}/../../../../../ REALPATH) +# set(DISTRIBUTION_REFROOT ${REFROOT}/ CACHE STRING "BB Dpkg root set from toolchain file location.") +# endif() +# endif() + +include("/sanitizers/BBToolchain64.cmake") + +if(DEFINED ENV{CC}) + set(CMAKE_C_COMPILER $ENV{CC} CACHE STRING "Instrumentation C compiler" FORCE) +endif() + +if(DEFINED ENV{CXX}) + set(CMAKE_CXX_COMPILER $ENV{CXX} CACHE STRING "Instrumentation C++ compiler" FORCE) +endif() + +set(LIBCXX_BUILD_PATH "$ENV{LIBCXX_BUILD_PATH}") + +# Force disabling the use of Readline. This is a holdover until readline builds with -fPIC. +# https://bbgithub.dev.bloomberg.com/swt/readline/issues/8 +set(BMQ_DISABLE_READLINE TRUE) + +set(TOOLCHAIN_CXX_FLAGS "${CMAKE_CXX_FLAGS_DEBUG}") +set(TOOLCHAIN_C_FLAGS "${CMAKE_C_FLAGS_DEBUG}") + +# Build shared flags. +string(CONCAT TOOLCHAIN_SHARED_FLAGS + "${}" + "${TOOLCHAIN_SHARED_FLAGS} " + "-O0 " + "-g " + "-fno-omit-frame-pointer " + "-fno-optimize-sibling-calls " + "-fdiagnostics-show-option " + "-isystem/usr/include " + ) + +# Apply shared flags to each language. +string(CONCAT TOOLCHAIN_CXX_FLAGS + "${TOOLCHAIN_CXX_FLAGS} " + "${TOOLCHAIN_SHARED_FLAGS} " + ) +string(CONCAT TOOLCHAIN_C_FLAGS + "${TOOLCHAIN_C_FLAGS} " + "${TOOLCHAIN_SHARED_FLAGS} " + ) + +# Use libc++ standard library for C++. +string(CONCAT TOOLCHAIN_CXX_FLAGS + "${TOOLCHAIN_CXX_FLAGS} " + "-stdlib=libc++ " + "-I${LIBCXX_BUILD_PATH}/include/c++/v1 " + ) + +# Suppress some warnings when building C++ code. +string(CONCAT TOOLCHAIN_CXX_FLAGS + "${TOOLCHAIN_CXX_FLAGS} " + "-Wno-c++98-compat " + "-Wno-c++98-compat-extra-semi " + "-Wno-c++98-compat-pedantic " + "-Wno-deprecated " + "-Wno-deprecated-declarations " + "-Wno-disabled-macro-expansion " + "-Wno-extra-semi-stmt " + "-Wno-inconsistent-missing-destructor-override " + "-Wno-inconsistent-missing-override " + "-Wno-old-style-cast " + "-Wno-undef " + "-Wno-zero-as-null-pointer-constant " + ) + +# Define linker flags (used for both shared-objects and executables). +string( CONCAT TOOLCHAIN_LINKER_FLAGS + "${CMAKE_LINKER_FLAGS_DEBUG}" + "-stdlib=libc++ " + "-L${LIBCXX_BUILD_PATH}/lib " + "-Wl,-rpath,${LIBCXX_BUILD_PATH}/lib " + "-lc++abi " + ) + +set( TOOLCHAIN_EXE_FLAGS "${TOOLCHAIN_LINKER_FLAGS}" ) + +macro(set_build_type type) + set(CMAKE_CXX_FLAGS_${type} + "${TOOLCHAIN_CXX_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) + set(CMAKE_C_FLAGS_${type} + "${TOOLCHAIN_C_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) + set(CMAKE_EXE_LINKER_FLAGS_${type} + "${TOOLCHAIN_EXE_FLAGS} ${TOOLCHAIN_${type}_FLAGS} -static-libsan" CACHE STRING "Default" FORCE) + set(CMAKE_SHARED_LINKER_FLAGS_${type} + "${TOOLCHAIN_LINKER_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) + set(CMAKE_MODULE_LINKER_FLAGS_${type} + "${TOOLCHAIN_LINKER_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) +endmacro() + +set( TOOLCHAIN_DEBUG_FLAGS + "-fsanitize=address " + ) + +# Set the final configuration variables, as understood by CMake. +set_build_type(DEBUG) + +# Disable GNU c++ extensions. +set(CMAKE_CXX_EXTENSIONS OFF) diff --git a/.github/workflows/sanitizers/clang-libcxx-msan.cmake b/.github/workflows/sanitizers/clang-libcxx-msan.cmake new file mode 100644 index 0000000000..bb45caf452 --- /dev/null +++ b/.github/workflows/sanitizers/clang-libcxx-msan.cmake @@ -0,0 +1,121 @@ +# Compiler-less toolchain for building with {Clang, libc++} + MemorySanitizer. +# The actual compiler is passed via `CXX` and `CC` environment variables. +#-=- + +cmake_minimum_required (VERSION 3.25) + +# if(NOT DEFINED DISTRIBUTION_REFROOT) +# if(DEFINED ENV{DISTRIBUTION_REFROOT}) +# set(DISTRIBUTION_REFROOT "$ENV{DISTRIBUTION_REFROOT}/" CACHE STRING "BB Dpkg root set from environment variable.") +# else() +# get_filename_component(REFROOT ${CMAKE_CURRENT_LIST_DIR}/../../../../../ REALPATH) +# set(DISTRIBUTION_REFROOT ${REFROOT}/ CACHE STRING "BB Dpkg root set from toolchain file location.") +# endif() +# endif() + +include("/sanitizers/BBToolchain64.cmake") + +if(DEFINED ENV{CC}) + set(CMAKE_C_COMPILER $ENV{CC} CACHE STRING "Instrumentation C compiler" FORCE) +endif() + +if(DEFINED ENV{CXX}) + set(CMAKE_CXX_COMPILER $ENV{CXX} CACHE STRING "Instrumentation C++ compiler" FORCE) +endif() + +set(LIBCXX_BUILD_PATH "$ENV{LIBCXX_BUILD_PATH}") + +# Force disabling the use of Readline. This is a holdover until readline builds with -fPIC. +# https://bbgithub.dev.bloomberg.com/swt/readline/issues/8 +set(BMQ_DISABLE_READLINE TRUE) + +set(TOOLCHAIN_CXX_FLAGS "${CMAKE_CXX_FLAGS_DEBUG}") +set(TOOLCHAIN_C_FLAGS "${CMAKE_C_FLAGS_DEBUG}") + +set(MSAN_SUPPRESSION_LIST_PATH "$ENV{DIR_SRC_BMQ}/etc/msansup.txt") + +# Build shared flags. +string(CONCAT TOOLCHAIN_SHARED_FLAGS + "${}" + "${TOOLCHAIN_SHARED_FLAGS} " + "-O0 " + "-g " + "-fno-omit-frame-pointer " + "-fdiagnostics-show-option " + "-fsanitize=memory " + "-fsanitize-blacklist=${MSAN_SUPPRESSION_LIST_PATH} " + "-isystem/usr/include " + ) + +# Apply shared flags to each language. +string(CONCAT TOOLCHAIN_CXX_FLAGS + "${TOOLCHAIN_CXX_FLAGS} " + "${TOOLCHAIN_SHARED_FLAGS} " + ) +string(CONCAT TOOLCHAIN_C_FLAGS + "${TOOLCHAIN_C_FLAGS} " + "${TOOLCHAIN_SHARED_FLAGS} " + ) + +# Use libc++ standard library for C++. +string(CONCAT TOOLCHAIN_CXX_FLAGS + "${TOOLCHAIN_CXX_FLAGS} " + "-stdlib=libc++ " + "-I${LIBCXX_BUILD_PATH}/include/c++/v1 " + ) + +# Suppress some warnings when building C++ code. +string(CONCAT TOOLCHAIN_CXX_FLAGS + "${TOOLCHAIN_CXX_FLAGS} " + "-Wno-c++98-compat " + "-Wno-c++98-compat-extra-semi " + "-Wno-c++98-compat-pedantic " + "-Wno-deprecated " + "-Wno-deprecated-declarations " + "-Wno-disabled-macro-expansion " + "-Wno-extra-semi-stmt " + "-Wno-inconsistent-missing-destructor-override " + "-Wno-inconsistent-missing-override " + "-Wno-old-style-cast " + "-Wno-undef " + "-Wno-zero-as-null-pointer-constant " + ) + +# Define linker flags (used for both shared-objects and executables). +string( CONCAT TOOLCHAIN_LINKER_FLAGS + "${CMAKE_LINKER_FLAGS_DEBUG}" + "-stdlib=libc++ " + "-L${LIBCXX_BUILD_PATH}/lib " + "-Wl,-rpath,${LIBCXX_BUILD_PATH}/lib " + "-lc++abi " + ) + +set( TOOLCHAIN_EXE_FLAGS "${TOOLCHAIN_LINKER_FLAGS}" ) + +macro(set_build_type type) + set(CMAKE_CXX_FLAGS_${type} + "${TOOLCHAIN_CXX_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) + set(CMAKE_C_FLAGS_${type} + "${TOOLCHAIN_C_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) + set(CMAKE_EXE_LINKER_FLAGS_${type} + "${TOOLCHAIN_EXE_FLAGS} ${TOOLCHAIN_${type}_FLAGS} -static-libsan" CACHE STRING "Default" FORCE) + set(CMAKE_SHARED_LINKER_FLAGS_${type} + "${TOOLCHAIN_LINKER_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) + set(CMAKE_MODULE_LINKER_FLAGS_${type} + "${TOOLCHAIN_LINKER_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) +endmacro() + +# Conditionally add flags helpful for debugging MemorySanitizer issues. +if (DEBUG_MEMORY_SANITIZER) + string(CONCAT TOOLCHAIN_DEBUG_FLAGS + "${TOOLCHAIN_SHARED_FLAGS} " + "-fsanitize-memory-track-origins=2 " + "-fno-optimize-sibling-calls " + ) +endif() + +# Set the final configuration variables, as understood by CMake. +set_build_type(DEBUG) + +# Disable GNU c++ extensions. +set(CMAKE_CXX_EXTENSIONS OFF) diff --git a/.github/workflows/sanitizers/sanitizers.json b/.github/workflows/sanitizers/sanitizers.json new file mode 100644 index 0000000000..f22525c8cf --- /dev/null +++ b/.github/workflows/sanitizers/sanitizers.json @@ -0,0 +1,48 @@ +{ + "asan": { + "bde_compiler_name": "asan", + "llvm_sanitizer_name": "Address", + "environment": { + "ASAN_OPTIONS": { + "external_symbolizer_path": "%%ROOT%%/_external/refroot/opt/bb/bin/llvm-symbolizer" + }, + "LSAN_OPTIONS": { + "log_threads": 1, + "verbosity": 0 + } + } + }, + "msan": { + "bde_compiler_name": "msan", + "llvm_sanitizer_name": "MemoryWithOrigins", + "llvm_specific_cmake_options": "-DLIBCXXABI_USE_LLVM_UNWINDER=OFF", + "environment": { + "MSAN_OPTIONS": { + "external_symbolizer_path": "%%ROOT%%/_external/refroot/opt/bb/bin/llvm-symbolizer" + } + } + }, + "tsan": { + "bde_compiler_name": "tsan", + "llvm_sanitizer_name": "Thread", + "environment": { + "TSAN_OPTIONS": { + "external_symbolizer_path": "%%ROOT%%/_external/refroot/opt/bb/bin/llvm-symbolizer", + "second_deadlock_stack": 1, + "suppressions": "%%SRC%%/etc/tsansup.txt" + } + } + }, + "ubsan": { + "bde_compiler_name": "ubsan", + "llvm_sanitizer_name": "Undefined", + "environment": { + "UBSAN_OPTIONS": { + "external_symbolizer_path": "%%ROOT%%/_external/refroot/opt/bb/bin/llvm-symbolizer", + "suppressions": "%%SRC%%/etc/ubsansup.txt", + "print_stacktrace": "1" + } + } + } + } + \ No newline at end of file diff --git a/.github/workflows/sanitizers/script.sh b/.github/workflows/sanitizers/script.sh new file mode 100644 index 0000000000..4d4cad13b1 --- /dev/null +++ b/.github/workflows/sanitizers/script.sh @@ -0,0 +1,267 @@ +#!/bin/bash + +set -eux + +# Prerequisites for LLVM installation +## NOTE: cmake : https://askubuntu.com/questions/355565/how-do-i-install-the-latest-version-of-cmake-from-the-command-line +apt update && apt install -y lsb-release wget software-properties-common gnupg git curl jq ninja-build bison libfl-dev pkg-config && rm -rf /var/lib/apt/lists/* /var/log/dpkg.log + +### NOTE: cmake : https://askubuntu.com/questions/355565/how-do-i-install-the-latest-version-of-cmake-from-the-command-line +wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null +apt-add-repository -y "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" +apt update && apt install -y cmake + +# FOR BMQ ONLY +# apt-get update && apt-get install -y bison libfl-dev pkg-config + +# Install LLVM +# wget https://apt.llvm.org/llvm.sh +chmod +x llvm.sh +LLVM_VERSION=18 +./llvm.sh ${LLVM_VERSION} all + +# Create version-agnostic pointers to required LLVM binaries. +ln -sf /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang +ln -sf /usr/bin/clang++-${LLVM_VERSION} /usr/bin/clang++ +ln -sf /usr/bin/llvm-symbolizer-${LLVM_VERSION} /usr/bin/llvm-symbolizer + +# Parse sanitizers config +cfgquery() { + jq "${1}" "sanitizers.json" --raw-output +} +SANITIZER_NAME="asan" +LLVM_SANITIZER_NAME="$(cfgquery .${SANITIZER_NAME}.llvm_sanitizer_name)" +# Check if llvm specific cmake options are present for the given sanitizer +LLVM_SPECIFIC_CMAKE_OPTIONS="$(cfgquery .${SANITIZER_NAME}.llvm_specific_cmake_options)" +if [[ "$LLVM_SPECIFIC_CMAKE_OPTIONS" == null ]]; then LLVM_SPECIFIC_CMAKE_OPTIONS=""; fi + +# Set some initial constants +PARALLELISM=2 +# PARALLELISM=8 + +DIR_ROOT="/bmq" +DIR_EXTERNAL="${DIR_ROOT}/_external" +DIR_SRCS_EXT="${DIR_EXTERNAL}/srcs" +DIR_BUILD_EXT="${DIR_SRCS_EXT}/cmake.bld" + +DIR_SRC_BMQ="${DIR_SRCS_EXT}/blazingmq" +DIR_BUILD_BMQ="${DIR_SRC_BMQ}/cmake.bld/Linux" + +# :: checkoutGitRepo() subroutine ::::::::::::::::::::::::::::::::::::::::::::: +checkoutGitRepo() { + local repo=$1 + local ref=$2 + local repoDir=$3 + echo "Checking out ${repo} at ${ref}" + + local repoPath="${DIR_SRCS_EXT}/${repoDir}" + # mkdir ${repoPath} + + git clone -b ${ref} ${repo} \ + --depth 1 --single-branch --no-tags -c advice.detachedHead=false "${repoPath}" + + # git -C "${repoPath}" clone -b ${ref} ${repo} \ + # --depth 1 --single-branch --no-tags -c advice.detachedHead=false +} +github_url() { echo "https://github.com/$1.git"; } + +# :: Download external dependencies ::::::::::::::::::::::::::::::: +mkdir -p ${DIR_SRCS_EXT} + +# TODO: Download BlazingMQ +BMQ_TAG="main" +checkoutGitRepo "$(github_url bloomberg/blazingmq)" "${BMQ_TAG}" "blazingmq" + +# Download LLVM +LLVM_TAG="llvmorg-18.1.8" +checkoutGitRepo "$(github_url llvm/llvm-project)" "${LLVM_TAG}" "llvm-project" + +# Download google-benchmark +GOOGLE_BENCHMARK_TAG="v1.8.4" +checkoutGitRepo "$(github_url google/benchmark)" "${GOOGLE_BENCHMARK_TAG}" "google-benchmark" + +# Download googletest +GOOGLETEST_TAG="v1.14.0" +checkoutGitRepo "$(github_url google/googletest)" "${GOOGLETEST_TAG}" "googletest" + +# Download zlib +ZLIB_TAG="v1.3.1" +checkoutGitRepo "$(github_url madler/zlib)" "${ZLIB_TAG}" "zlib" + +# Download BlazingMQ dependencies +pushd "${DIR_EXTERNAL}" +/sanitizers/build_deps.sh only-download +popd + + +# :: Build libc++ with required instrumentation ::::::::::::::::::::::::::::::: +# +# The extent to which all dependencies to be compiled with sanitizer-support +# varies by sanitizer. MemorySanitizer is especially unforgiving: Failing to +# link against an instrumented standard library will yield many false +# positives. Concensus is that compiling libc++ with `-fsanitize=memory` is a +# significantly easier endeavor than doing the same with libstdc++ (the gcc +# standard library). +# +# We therefore opt to use libc++ here, just to ensure maximum flexibility. We +# follow build instructions from https://libcxx.llvm.org/BuildingLibcxx.html +LIBCXX_SRC_PATH="${DIR_SRCS_EXT}/llvm-project/runtimes" +LIBCXX_BUILD_PATH="${LIBCXX_SRC_PATH}/cmake.bld" + +cmake -B "${LIBCXX_BUILD_PATH}" \ + -S "${LIBCXX_SRC_PATH}" \ + -DCMAKE_BUILD_TYPE="Debug" \ + -DCMAKE_C_COMPILER="clang" \ + -DCMAKE_CXX_COMPILER="clang++" \ + -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ + -DLLVM_USE_SANITIZER="${LLVM_SANITIZER_NAME}" \ + ${LLVM_SPECIFIC_CMAKE_OPTIONS} + +cmake --build "${LIBCXX_BUILD_PATH}" -j${PARALLELISM} --target cxx cxxabi unwind generate-cxx-headers + +# Variables read by our custom CMake toolchain used to build everything else. +export LIBCXX_BUILD_PATH="$(realpath ${LIBCXX_BUILD_PATH})" +export DIR_SRC_BMQ="${DIR_SRC_BMQ}" + +# :: Build BDE + NTF :::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +TOOLCHAIN_PATH="/sanitizers/clang-libcxx-${SANITIZER_NAME}.cmake" +export CC="clang" +export CXX="clang++" +export BBS_BUILD_SYSTEM="ON" +PATH="$PATH:$(realpath ${DIR_SRCS_EXT}/bde-tools/bin)" +export PATH + +pushd ${DIR_SRCS_EXT} + +pushd "bde" +eval "$(bbs_build_env -u dbg_64_safe_cpp20 -b "${DIR_BUILD_EXT}/bde")" +bbs_build configure --toolchain "${TOOLCHAIN_PATH}" +bbs_build build -j${PARALLELISM} +bbs_build --install=/opt/bb --prefix=/ install +popd + +pushd "ntf-core" +# TODO The deprecated flag "-fcoroutines-ts" has been removed in clang +# 17.0.1, but NTF is still using it. We manually change this flag until +# the fix in issue 175307231 is resolved. +sed -i 's/fcoroutines-ts/fcoroutines/g' 'repository.cmake' + +./configure --keep \ + --prefix /opt/bb \ + --output "${DIR_BUILD_EXT}/ntf" \ + --without-warnings-as-errors \ + --without-usage-examples \ + --without-applications \ + --ufid 'dbg_64_safe_cpp20' \ + --toolchain "${TOOLCHAIN_PATH}" +make -j${PARALLELISM} +make install +popd + +# Note: Hack to circumvent faulty behavior in "nts-targets.cmake" +ln -sf "/opt/bb/include" "/opt/include" +ln -sf "/opt/bb/lib64" "/opt/lib64" + +# pushd DIR_SRCS_EXT +popd + +# :: Setup CMake options for all remaining builds ::::::::::::::::::::::::::::: +CMAKE_OPTIONS="\ + -D BUILD_BITNESS=64 \ + -D CMAKE_BUILD_TYPE=Debug \ + -D CMAKE_INSTALL_INCLUDEDIR=include \ + -D CMAKE_INSTALL_LIBDIR=lib64 \ + -D CMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_PATH}" + +# :: Build GoogleTest ::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +cmake -B "${DIR_SRCS_EXT}/googletest/cmake.bld" \ + -S "${DIR_SRCS_EXT}/googletest" ${CMAKE_OPTIONS} \ + -DCMAKE_INSTALL_PREFIX=/opt/bb +cmake --build "${DIR_SRCS_EXT}/googletest/cmake.bld" -j${PARALLELISM} +cmake --install "${DIR_SRCS_EXT}/googletest/cmake.bld" --prefix "/opt/bb" + + +# :: Build Google Benchmark ::::::::::::::::::::::::::::::::::::::::::::::::::: +cmake -B "${DIR_SRCS_EXT}/google-benchmark/cmake.bld" \ + -S "${DIR_SRCS_EXT}/google-benchmark" ${CMAKE_OPTIONS} \ + -DCMAKE_INSTALL_PREFIX=/opt/bb \ + -DBENCHMARK_DOWNLOAD_DEPENDENCIES="ON" \ + -DBENCHMARK_ENABLE_GTEST_TESTS="false" \ + -DBENCHMARK_ENABLE_TESTING="OFF" +cmake --build "${DIR_SRCS_EXT}/google-benchmark/cmake.bld" -j${PARALLELISM} +cmake --install "${DIR_SRCS_EXT}/google-benchmark/cmake.bld" --prefix "/opt/bb" + +## :: Build zlib :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +# Note: zlib has completely broken CMake install rules, so we must +# specify the install prefix *exactly* as it will be at configure +# time +# https://discourse.cmake.org/t/cmake-install-prefix-not-work/5040 +cmake -B "${DIR_SRCS_EXT}/zlib/cmake.bld" -S "${DIR_SRCS_EXT}/zlib" \ + -D CMAKE_INSTALL_PREFIX="/opt/bb" \ + ${CMAKE_OPTIONS} +# Make and install zlib. +cmake --build "${DIR_SRCS_EXT}/zlib/cmake.bld" -j${PARALLELISM} +cmake --install "${DIR_SRCS_EXT}/zlib/cmake.bld" + +# :: Build BlazingMQ :::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +# cmake -B "${DIR_BUILD_BMQ}" -S "${DIR_SRC_BMQ}" \ +# -DBDE_BUILD_TARGET_SAFE=1 ${CMAKE_OPTIONS} +# cmake --build "${DIR_BUILD_BMQ}" -j${PARALLELISM} \ +# --target all.t -v --clean-first + +PKG_CONFIG_PATH="/opt/bb/lib64/pkgconfig:/opt/bb/lib/pkgconfig:/opt/bb/share/pkgconfig:$(pkg-config --variable pc_path pkg-config)" \ +cmake -B "${DIR_BUILD_BMQ}" -S "${DIR_SRC_BMQ}" -G Ninja \ + -DBDE_BUILD_TARGET_64=ON \ + -DBDE_BUILD_TARGET_CPP17=ON \ + -DCMAKE_PREFIX_PATH="${DIR_SRCS_EXT}/bde-tools/BdeBuildSystem" \ + -DBDE_BUILD_TARGET_SAFE=1 ${CMAKE_OPTIONS} +cmake --build "${DIR_BUILD_BMQ}" -j${PARALLELISM} \ + --target all.t -v --clean-first + + +# cmake -S . -B build/blazingmq -G Ninja \ +# PKG_CONFIG_PATH="/opt/bb/lib64/pkgconfig:/opt/bb/lib/pkgconfig:$(pkg-config --variable pc_path pkg-config)" \ +# cmake -B "${DIR_BUILD_BMQ}" -S "${DIR_SRC_BMQ}" -G Ninja \ +# -DCMAKE_TOOLCHAIN_FILE="${DIR_SRCS_EXT}/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake" \ +# -DCMAKE_BUILD_TYPE=Debug \ +# -DBDE_BUILD_TARGET_SAFE=ON \ +# -DBDE_BUILD_TARGET_64=ON \ +# -DBDE_BUILD_TARGET_CPP17=ON \ +# -DCMAKE_MODULE_PATH="${DIR_SRCS_EXT}/bde-tools/cmake;${DIR_SRCS_EXT}/bde-tools/BdeBuildSystem" \ +# -DCMAKE_PREFIX_PATH="${DIR_SRCS_EXT}/bde-tools/BdeBuildSystem" \ +# -DCMAKE_CXX_STANDARD=17 \ +# -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ +# -DFLEX_ROOT=/usr/lib/x86_64-linux-gnu \ +# -DCMAKE_INSTALL_LIBDIR=lib64 + # cmake --build build/blazingmq --parallel 8 --target all + + + # # -DBDE_BUILD_TARGET_64=1 \ + # # -DCMAKE_BUILD_TYPE=Debug \ + # # -DCMAKE_INSTALL_LIBDIR="lib" \ + # -DCMAKE_INSTALL_PREFIX="${DIR_INSTALL}" \ + # -DCMAKE_MODULE_PATH="${DIR_THIRDPARTY}/bde-tools/cmake;${DIR_THIRDPARTY}/bde-tools/BdeBuildSystem" \ + # -DCMAKE_PREFIX_PATH="${DIR_INSTALL}" \ + # -DCMAKE_TOOLCHAIN_FILE="${DIR_THIRDPARTY}/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake" \ + # -DCMAKE_CXX_STANDARD=17 \ + # -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ + # -DFLEX_ROOT=/usr/lib/x86_64-linux-gnu) + +################################################ +# WORKS!!! +# PATH="${DIR_SRCS_EXT}/bde-tools/bin:$PATH" + +# CMAKE_OPTIONS=(\ +# -DBDE_BUILD_TARGET_64=1 \ +# -DCMAKE_BUILD_TYPE=Debug \ +# -DCMAKE_INSTALL_LIBDIR="lib" \ +# -DCMAKE_MODULE_PATH="${DIR_SRCS_EXT}/bde-tools/cmake;${DIR_SRCS_EXT}/bde-tools/BdeBuildSystem" \ +# -DCMAKE_TOOLCHAIN_FILE="${DIR_SRCS_EXT}/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake" \ +# -DCMAKE_CXX_STANDARD=17 \ +# -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ +# -DFLEX_ROOT=/usr/lib/x86_64-linux-gnu) + +# PKG_CONFIG_PATH="/opt/bb/lib64/pkgconfig:/opt/bb/lib/pkgconfig:/opt/bb/share/pkgconfig:$(pkg-config --variable pc_path pkg-config)" \ +# cmake -B "${DIR_BUILD_BMQ}" -S "${DIR_SRC_BMQ}" "${CMAKE_OPTIONS[@]}" + + From aaba5ac8c563d0c104b661f28b84fff77f1a7880 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Mon, 8 Jul 2024 16:47:48 +0300 Subject: [PATCH 004/105] Fix build errors, add run test script Signed-off-by: Aleksandr Ivanov --- .../sanitizers/clang-libcxx-asan.cmake | 1 + .../sanitizers/clang-libcxx-msan.cmake | 1 + .github/workflows/sanitizers/sanitizers.json | 81 ++++++++-------- .github/workflows/sanitizers/script.sh | 96 ++++++++----------- 4 files changed, 82 insertions(+), 97 deletions(-) diff --git a/.github/workflows/sanitizers/clang-libcxx-asan.cmake b/.github/workflows/sanitizers/clang-libcxx-asan.cmake index 1230eba33e..eda60f2267 100644 --- a/.github/workflows/sanitizers/clang-libcxx-asan.cmake +++ b/.github/workflows/sanitizers/clang-libcxx-asan.cmake @@ -42,6 +42,7 @@ string(CONCAT TOOLCHAIN_SHARED_FLAGS "-fno-optimize-sibling-calls " "-fdiagnostics-show-option " "-isystem/usr/include " + "-isystem/usr/lib/llvm-18/lib/clang/18/include " ) # Apply shared flags to each language. diff --git a/.github/workflows/sanitizers/clang-libcxx-msan.cmake b/.github/workflows/sanitizers/clang-libcxx-msan.cmake index bb45caf452..e93abde3b1 100644 --- a/.github/workflows/sanitizers/clang-libcxx-msan.cmake +++ b/.github/workflows/sanitizers/clang-libcxx-msan.cmake @@ -45,6 +45,7 @@ string(CONCAT TOOLCHAIN_SHARED_FLAGS "-fsanitize=memory " "-fsanitize-blacklist=${MSAN_SUPPRESSION_LIST_PATH} " "-isystem/usr/include " + "-isystem/usr/lib/llvm-18/lib/clang/18/include " ) # Apply shared flags to each language. diff --git a/.github/workflows/sanitizers/sanitizers.json b/.github/workflows/sanitizers/sanitizers.json index f22525c8cf..d50a4ca20a 100644 --- a/.github/workflows/sanitizers/sanitizers.json +++ b/.github/workflows/sanitizers/sanitizers.json @@ -1,48 +1,47 @@ { - "asan": { - "bde_compiler_name": "asan", - "llvm_sanitizer_name": "Address", - "environment": { - "ASAN_OPTIONS": { - "external_symbolizer_path": "%%ROOT%%/_external/refroot/opt/bb/bin/llvm-symbolizer" - }, - "LSAN_OPTIONS": { - "log_threads": 1, - "verbosity": 0 - } + "asan": { + "bde_compiler_name": "asan", + "llvm_sanitizer_name": "Address", + "environment": { + "ASAN_OPTIONS": { + "external_symbolizer_path": "/usr/bin/llvm-symbolizer" + }, + "LSAN_OPTIONS": { + "log_threads": 1, + "verbosity": 0 } - }, - "msan": { - "bde_compiler_name": "msan", - "llvm_sanitizer_name": "MemoryWithOrigins", - "llvm_specific_cmake_options": "-DLIBCXXABI_USE_LLVM_UNWINDER=OFF", - "environment": { - "MSAN_OPTIONS": { - "external_symbolizer_path": "%%ROOT%%/_external/refroot/opt/bb/bin/llvm-symbolizer" - } + } + }, + "msan": { + "bde_compiler_name": "msan", + "llvm_sanitizer_name": "MemoryWithOrigins", + "llvm_specific_cmake_options": "-DLIBCXXABI_USE_LLVM_UNWINDER=OFF", + "environment": { + "MSAN_OPTIONS": { + "external_symbolizer_path": "/usr/bin/llvm-symbolizer" } - }, - "tsan": { - "bde_compiler_name": "tsan", - "llvm_sanitizer_name": "Thread", - "environment": { - "TSAN_OPTIONS": { - "external_symbolizer_path": "%%ROOT%%/_external/refroot/opt/bb/bin/llvm-symbolizer", - "second_deadlock_stack": 1, - "suppressions": "%%SRC%%/etc/tsansup.txt" - } + } + }, + "tsan": { + "bde_compiler_name": "tsan", + "llvm_sanitizer_name": "Thread", + "environment": { + "TSAN_OPTIONS": { + "external_symbolizer_path": "/usr/bin/llvm-symbolizer", + "second_deadlock_stack": 1, + "suppressions": "%%SRC%%/etc/tsansup.txt" } - }, - "ubsan": { - "bde_compiler_name": "ubsan", - "llvm_sanitizer_name": "Undefined", - "environment": { - "UBSAN_OPTIONS": { - "external_symbolizer_path": "%%ROOT%%/_external/refroot/opt/bb/bin/llvm-symbolizer", - "suppressions": "%%SRC%%/etc/ubsansup.txt", - "print_stacktrace": "1" - } + } + }, + "ubsan": { + "bde_compiler_name": "ubsan", + "llvm_sanitizer_name": "Undefined", + "environment": { + "UBSAN_OPTIONS": { + "external_symbolizer_path": "/usr/bin/llvm-symbolizer", + "suppressions": "%%SRC%%/etc/ubsansup.txt", + "print_stacktrace": "1" } } } - \ No newline at end of file +} diff --git a/.github/workflows/sanitizers/script.sh b/.github/workflows/sanitizers/script.sh index 4d4cad13b1..aedd1ea5e8 100644 --- a/.github/workflows/sanitizers/script.sh +++ b/.github/workflows/sanitizers/script.sh @@ -9,10 +9,9 @@ apt update && apt install -y lsb-release wget software-properties-common gnupg g ### NOTE: cmake : https://askubuntu.com/questions/355565/how-do-i-install-the-latest-version-of-cmake-from-the-command-line wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null apt-add-repository -y "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" -apt update && apt install -y cmake -# FOR BMQ ONLY -# apt-get update && apt-get install -y bison libfl-dev pkg-config +# CMAKE and FOR BMQ ONLY +apt update && apt install -y cmake bison libfl-dev pkg-config # Install LLVM # wget https://apt.llvm.org/llvm.sh @@ -29,14 +28,14 @@ ln -sf /usr/bin/llvm-symbolizer-${LLVM_VERSION} /usr/bin/llvm-symbolizer cfgquery() { jq "${1}" "sanitizers.json" --raw-output } -SANITIZER_NAME="asan" +SANITIZER_NAME="msan" LLVM_SANITIZER_NAME="$(cfgquery .${SANITIZER_NAME}.llvm_sanitizer_name)" # Check if llvm specific cmake options are present for the given sanitizer LLVM_SPECIFIC_CMAKE_OPTIONS="$(cfgquery .${SANITIZER_NAME}.llvm_specific_cmake_options)" if [[ "$LLVM_SPECIFIC_CMAKE_OPTIONS" == null ]]; then LLVM_SPECIFIC_CMAKE_OPTIONS=""; fi # Set some initial constants -PARALLELISM=2 +PARALLELISM=4 # PARALLELISM=8 DIR_ROOT="/bmq" @@ -59,9 +58,6 @@ checkoutGitRepo() { git clone -b ${ref} ${repo} \ --depth 1 --single-branch --no-tags -c advice.detachedHead=false "${repoPath}" - - # git -C "${repoPath}" clone -b ${ref} ${repo} \ - # --depth 1 --single-branch --no-tags -c advice.detachedHead=false } github_url() { echo "https://github.com/$1.git"; } @@ -204,10 +200,6 @@ cmake --build "${DIR_SRCS_EXT}/zlib/cmake.bld" -j${PARALLELISM} cmake --install "${DIR_SRCS_EXT}/zlib/cmake.bld" # :: Build BlazingMQ :::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -# cmake -B "${DIR_BUILD_BMQ}" -S "${DIR_SRC_BMQ}" \ -# -DBDE_BUILD_TARGET_SAFE=1 ${CMAKE_OPTIONS} -# cmake --build "${DIR_BUILD_BMQ}" -j${PARALLELISM} \ -# --target all.t -v --clean-first PKG_CONFIG_PATH="/opt/bb/lib64/pkgconfig:/opt/bb/lib/pkgconfig:/opt/bb/share/pkgconfig:$(pkg-config --variable pc_path pkg-config)" \ cmake -B "${DIR_BUILD_BMQ}" -S "${DIR_SRC_BMQ}" -G Ninja \ @@ -218,50 +210,42 @@ cmake -B "${DIR_BUILD_BMQ}" -S "${DIR_SRC_BMQ}" -G Ninja \ cmake --build "${DIR_BUILD_BMQ}" -j${PARALLELISM} \ --target all.t -v --clean-first +# :: Create testing scripts ::::::::::::::::::::::::::::::::::::::::::::::::::: +envcfgquery() { + # Parses the '.environment' object from 'sanitizers.json', + # and outputs a string of whitespace-separated 'VAR=VAL' pairs intended to + # be used to set the environment for a command. + # e.g. 'asan' -> 'ASAN_OPTIONS="foo=bar:baz=baf" LSAN_OPTIONS="abc=fgh"' + # + echo $(cfgquery " \ + .${1}.environment | \ + to_entries | \ + map(\"\(.key)=\\\"\(.value | \ + to_entries | \ + map(\"\(.key)=\(.value)\") | \ + join(\":\"))\\\"\") | \ + join(\" \")") | + sed "s|%%SRC%%|$(realpath ${DIR_SRC_BMQ})|g" | + sed "s|%%ROOT%%|$(realpath ${DIR_ROOT})|g" +} + +mkscript() { + local cmd=${1} + local outfile=${2} + + echo '#!/usr/bin/env bash' > ${outfile} + echo "${cmd}" >> ${outfile} + chmod +x ${outfile} +} -# cmake -S . -B build/blazingmq -G Ninja \ -# PKG_CONFIG_PATH="/opt/bb/lib64/pkgconfig:/opt/bb/lib/pkgconfig:$(pkg-config --variable pc_path pkg-config)" \ -# cmake -B "${DIR_BUILD_BMQ}" -S "${DIR_SRC_BMQ}" -G Ninja \ -# -DCMAKE_TOOLCHAIN_FILE="${DIR_SRCS_EXT}/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake" \ -# -DCMAKE_BUILD_TYPE=Debug \ -# -DBDE_BUILD_TARGET_SAFE=ON \ -# -DBDE_BUILD_TARGET_64=ON \ -# -DBDE_BUILD_TARGET_CPP17=ON \ -# -DCMAKE_MODULE_PATH="${DIR_SRCS_EXT}/bde-tools/cmake;${DIR_SRCS_EXT}/bde-tools/BdeBuildSystem" \ -# -DCMAKE_PREFIX_PATH="${DIR_SRCS_EXT}/bde-tools/BdeBuildSystem" \ -# -DCMAKE_CXX_STANDARD=17 \ -# -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ -# -DFLEX_ROOT=/usr/lib/x86_64-linux-gnu \ -# -DCMAKE_INSTALL_LIBDIR=lib64 - # cmake --build build/blazingmq --parallel 8 --target all - - - # # -DBDE_BUILD_TARGET_64=1 \ - # # -DCMAKE_BUILD_TYPE=Debug \ - # # -DCMAKE_INSTALL_LIBDIR="lib" \ - # -DCMAKE_INSTALL_PREFIX="${DIR_INSTALL}" \ - # -DCMAKE_MODULE_PATH="${DIR_THIRDPARTY}/bde-tools/cmake;${DIR_THIRDPARTY}/bde-tools/BdeBuildSystem" \ - # -DCMAKE_PREFIX_PATH="${DIR_INSTALL}" \ - # -DCMAKE_TOOLCHAIN_FILE="${DIR_THIRDPARTY}/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake" \ - # -DCMAKE_CXX_STANDARD=17 \ - # -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ - # -DFLEX_ROOT=/usr/lib/x86_64-linux-gnu) - -################################################ -# WORKS!!! -# PATH="${DIR_SRCS_EXT}/bde-tools/bin:$PATH" - -# CMAKE_OPTIONS=(\ -# -DBDE_BUILD_TARGET_64=1 \ -# -DCMAKE_BUILD_TYPE=Debug \ -# -DCMAKE_INSTALL_LIBDIR="lib" \ -# -DCMAKE_MODULE_PATH="${DIR_SRCS_EXT}/bde-tools/cmake;${DIR_SRCS_EXT}/bde-tools/BdeBuildSystem" \ -# -DCMAKE_TOOLCHAIN_FILE="${DIR_SRCS_EXT}/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake" \ -# -DCMAKE_CXX_STANDARD=17 \ -# -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ -# -DFLEX_ROOT=/usr/lib/x86_64-linux-gnu) - -# PKG_CONFIG_PATH="/opt/bb/lib64/pkgconfig:/opt/bb/lib/pkgconfig:/opt/bb/share/pkgconfig:$(pkg-config --variable pc_path pkg-config)" \ -# cmake -B "${DIR_BUILD_BMQ}" -S "${DIR_SRC_BMQ}" "${CMAKE_OPTIONS[@]}" +SANITIZER_ENV="BMQ_BUILD=$(realpath ${DIR_BUILD_BMQ}) " +SANITIZER_ENV+="BMQ_REPO=${DIR_SRC_BMQ} " +SANITIZER_ENV+="$(envcfgquery ${SANITIZER_NAME})" +# 'run-env.sh' runs a command with environment required of the sanitizer. +mkscript "${SANITIZER_ENV} \${@}" "${DIR_BUILD_BMQ}/run-env.sh" +# 'run-unittests.sh' runs all instrumented unit-tests. +CMD="cd $(realpath ${DIR_SRC_BMQ}) && " +CMD+="${DIR_BUILD_BMQ}/run-env.sh ctest -E mwcsys_executil.t --output-on-failure" +mkscript "${CMD}" "${DIR_BUILD_BMQ}/run-unittests.sh" From 79ec4bc12169512571d7fd97360ffdb60b218d78 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 10:58:01 +0300 Subject: [PATCH 005/105] build.yaml: add build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/build.yaml | 305 ++--------------------------------- 1 file changed, 9 insertions(+), 296 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2f83ddce62..3319aa5517 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -4,6 +4,7 @@ on: push: branches: - main + - add_sanitizers_to_ci pull_request: branches: - main @@ -58,8 +59,8 @@ jobs: path: deps key: deps-${{ steps.get-hash.outputs.deps_hash }} - build_ubuntu: - name: Build [ubuntu] + build_and_run_asan: + name: Build and run AddressSanitizer runs-on: ubuntu-latest needs: build_dependencies steps: @@ -71,297 +72,9 @@ jobs: with: path: deps key: deps-${{ steps.get-hash.outputs.deps_hash }} - - name: Set up dependencies - run: | - sudo apt-get update - sudo apt-get install -qy build-essential \ - gdb \ - curl \ - python3.10 \ - python3-pip \ - cmake \ - ninja-build \ - pkg-config \ - bison \ - libfl-dev \ - libbenchmark-dev \ - libgmock-dev \ - libz-dev - - name: Install cached non packaged dependencies - working-directory: deps - run: ../docker/build_deps.sh - - name: Build BlazingMQ - env: - PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig - run: | - cmake -S . -B build/blazingmq -G Ninja \ - -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake \ - -DCMAKE_BUILD_TYPE=Debug \ - -DBDE_BUILD_TARGET_SAFE=ON \ - -DBDE_BUILD_TARGET_64=ON \ - -DBDE_BUILD_TARGET_CPP17=ON \ - -DCMAKE_PREFIX_PATH=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem \ - -DCMAKE_INSTALL_LIBDIR=lib64 - cmake --build build/blazingmq --parallel 8 --target all - - name: Clean-up build directories before caching - run: | - find . -name "*.o" -type f -delete - find . -name "*.a" -type f -delete - - uses: actions/cache@v4 - with: - path: | - build/blazingmq - deps - /opt/bb/include - key: cache-${{ github.sha }} - - unit_tests_cxx: - name: UT [c++] - runs-on: ubuntu-latest - needs: build_dependencies - steps: - - uses: actions/checkout@v4 - - name: Get dependencies hash - id: get-hash - run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT - - uses: actions/cache/restore@v4 - with: - path: deps - key: deps-${{ steps.get-hash.outputs.deps_hash }} - - name: Set up dependencies - run: | - sudo apt-get update - sudo apt-get install -qy build-essential \ - gdb \ - curl \ - python3.10 \ - python3-pip \ - cmake \ - ninja-build \ - pkg-config \ - bison \ - libfl-dev \ - libbenchmark-dev \ - libgmock-dev \ - libz-dev - - name: Install cached non packaged dependencies - working-directory: deps - run: ../docker/build_deps.sh - - name: Build BlazingMQ UTs - env: - PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig - run: | - cmake -S . -B build/blazingmq -G Ninja \ - -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake \ - -DCMAKE_BUILD_TYPE=Debug \ - -DBDE_BUILD_TARGET_SAFE=ON \ - -DBDE_BUILD_TARGET_64=ON \ - -DBDE_BUILD_TARGET_CPP17=ON \ - -DCMAKE_PREFIX_PATH=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem \ - -DCMAKE_INSTALL_LIBDIR=lib64 - cmake --build build/blazingmq --parallel 8 --target all.t - - name: Run C++ Unit Tests - run: | - cd ${{ github.workspace }}/build/blazingmq - ctest -E mwcsys_executil.t --output-on-failure - - unit_tests_python: - name: UT [python] - runs-on: ubuntu-latest - needs: build_ubuntu - steps: - - uses: actions/checkout@v4 - - uses: actions/cache/@v4 - with: - path: | - build/blazingmq - deps - /opt/bb/include - key: cache-${{ github.sha }} - - name: Run Python Unit Tests - env: - PYTHONPATH: ${{ github.workspace }}/src/python - run: | - pip install -r ${{ github.workspace }}/src/python/requirements.txt - pip install --force-reinstall ruff==0.4.10 - cd ${{ github.workspace }} - src/python/bin/schemagen - src/python/bin/tweakgen - pytest src/python - - integration_tests_ubuntu: - name: IT [${{ matrix.cluster }}/${{ matrix.mode }}] - strategy: - matrix: - mode: ["legacy_mode", "fsm_mode"] - cluster: ["single", "multi"] - fail-fast: false - runs-on: ubuntu-latest - needs: build_ubuntu - steps: - - uses: actions/checkout@v4 - - uses: actions/cache/@v4 - with: - path: | - build/blazingmq - deps - /opt/bb/include - key: cache-${{ github.sha }} - - name: Run Integration Tests - run: | - pip install -r ${{ github.workspace }}/src/python/requirements.txt - ${{ github.workspace }}/src/integration-tests/run-tests "${{ matrix.mode }} and ${{ matrix.cluster }}" \ - --log-level ERROR \ - --log-file-level=info \ - --bmq-tolerate-dirty-shutdown \ - --bmq-log-dir=failure-logs \ - --bmq-log-level=INFO \ - --junitxml=integration-tests.xml \ - --tb long \ - --reruns=3 \ - -n 4 -v - - fuzz_tests_ubuntu: - name: Fuzz test [${{ matrix.request }}] - strategy: - matrix: - request: ["identity", "open_queue", "configure_queue_stream", "put", "confirm", "close_queue", "disconnect"] - fail-fast: false - runs-on: ubuntu-latest - needs: build_ubuntu - steps: - - uses: actions/checkout@v4 - - uses: actions/cache/@v4 - with: - path: | - build/blazingmq - deps - /opt/bb/include - key: cache-${{ github.sha }} - - name: Run Fuzz Test - run: | - pip install -r ${{ github.workspace }}/src/python/requirements.txt - cd src/python - python3 -m blazingmq.dev.fuzztest --broker-dir ${{ github.workspace }}/build/blazingmq/src/applications/bmqbrkr --request ${{ matrix.request }} - - build_macosx: - name: Build [macosx_${{ matrix.arch }}] - runs-on: ${{ matrix.os }} - strategy: - matrix: - include: - - os: macos-14 - arch: arm64 - steps: - - uses: actions/checkout@v4 - - name: Set up dependencies - run: | - brew install \ - curl \ - python@3.10 \ - ninja \ - bison \ - flex \ - google-benchmark - - name: Build BlazingMQ - run: bin/build-darwin.sh - - build_and_test_bmqprometheus_plugin: - name: "Build Prometheus plugin [ubuntu]" - runs-on: ubuntu-latest - needs: build_ubuntu - steps: - - uses: actions/checkout@v4 - - uses: actions/cache/@v4 - with: - path: | - build/blazingmq - deps - /opt/bb/include - key: cache-${{ github.sha }} - - name: Set up plugins dependencies - run: | - sudo apt-get update - sudo apt-get install -qy build-essential \ - gdb \ - curl \ - python3.10 \ - cmake \ - ninja-build \ - pkg-config \ - bison \ - libfl-dev \ - libbenchmark-dev \ - libgmock-dev \ - libz-dev \ - autoconf \ - libtool - - name: Create dependency fetcher working directory - run: mkdir -p deps - - name: Fetch & Build non packaged plugins dependencies - working-directory: deps - run: ${{ github.workspace }}/src/plugins/bmqprometheus/build_prometheus_deps.sh - - name: Build plugins - env: - PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig - run: | - cmake -S . -B build/blazingmq -G Ninja \ - -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake \ - -DCMAKE_BUILD_TYPE=Debug \ - -DBDE_BUILD_TARGET_SAFE=ON \ - -DBDE_BUILD_TARGET_64=ON \ - -DBDE_BUILD_TARGET_CPP17=ON \ - -DCMAKE_PREFIX_PATH=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem \ - -DCMAKE_INSTALL_LIBDIR=lib64 \ - -DINSTALL_TARGETS=prometheus - cmake --build build/blazingmq --parallel 8 --target all - - name: Create prometheus dir - run: mkdir -p prometheus_dir - - name: Download Prometheus - run: curl -SL "https://github.com/prometheus/prometheus/releases/download/v2.45.1/prometheus-2.45.1.linux-amd64.tar.gz" | tar -xzC prometheus_dir/ - - name: Run Prometheus - run: ./prometheus_dir/prometheus-2.45.1.linux-amd64/prometheus --config.file=${{ github.workspace }}/src/plugins/bmqprometheus/tests/prometheus_localhost.yaml --web.enable-lifecycle --storage.tsdb.path=${{ github.workspace }}/prometheus_dir/data & - - name: Run Pushgateway - run: | - docker pull prom/pushgateway - docker run -d -p 9091:9091 prom/pushgateway - - name: Run BMQPrometheus plugin integration test in "pull" mode - run: ${{ github.workspace }}/src/plugins/bmqprometheus/tests/bmqprometheus_prometheusstatconsumer_test.py -p ${{ github.workspace }}/build/blazingmq -m pull --no-docker - - name: Clear Prometheus database and restart - run: | - curl -X POST "http://localhost:9090/-/quit" - rm -rf prometheus_dir/data - ./prometheus_dir/prometheus-2.45.1.linux-amd64/prometheus --config.file=${{ github.workspace }}/src/plugins/bmqprometheus/tests/prometheus_localhost.yaml --web.enable-lifecycle --storage.tsdb.path=${{ github.workspace }}/prometheus_dir/data & - - name: Run Prometheus plugin integration test in "push" mode - run: ${{ github.workspace }}/src/plugins/bmqprometheus/tests/bmqprometheus_prometheusstatconsumer_test.py -p ${{ github.workspace }}/build/blazingmq -m push --no-docker - - documentation: - name: "Documentation" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up dependencies - run: | - sudo apt-get update - sudo apt-get install -qy doxygen - - name: Build docs - run: | - doxygen Doxyfile - - shellcheck: - name: Shellcheck - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Run ShellCheck - uses: ludeeus/action-shellcheck@master - - docker_build_ubuntu: - name: "Docker [ubuntu]" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Docker Single-Node Workflow - run: docker compose -f docker/single-node/docker-compose.yaml up --build -d - - name: Docker Cluster Workflow - run: docker compose -f docker/cluster/docker-compose.yaml up --build -d + - name: List deps + run: ls -lah deps + - name: List deps/bde + run: ls -lah deps/bde + - name: List deps/ntf-core + run: ls -lah deps/bde-core From 40a71cd718d415167b466ab35179d58fdec236a5 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 11:03:06 +0300 Subject: [PATCH 006/105] build.yaml: fix build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/build.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 3319aa5517..3a5c7cdc88 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -73,8 +73,8 @@ jobs: path: deps key: deps-${{ steps.get-hash.outputs.deps_hash }} - name: List deps - run: ls -lah deps + run: ls -lah ${{ github.workspace }}/deps - name: List deps/bde - run: ls -lah deps/bde + run: ls -lah ${{ github.workspace }}/deps/bde - name: List deps/ntf-core - run: ls -lah deps/bde-core + run: ls -lah ${{ github.workspace }}/deps/ntf-core From e0d420ad6a7f13853d3af1e40e8252b901ac2e32 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 11:13:15 +0300 Subject: [PATCH 007/105] build.yaml: debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/build.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 3a5c7cdc88..36990decd9 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -74,7 +74,9 @@ jobs: key: deps-${{ steps.get-hash.outputs.deps_hash }} - name: List deps run: ls -lah ${{ github.workspace }}/deps - - name: List deps/bde - run: ls -lah ${{ github.workspace }}/deps/bde - - name: List deps/ntf-core - run: ls -lah ${{ github.workspace }}/deps/ntf-core + - name: List deps/srsc/bde + run: ls -lah ${{ github.workspace }}/deps/srsc/bde + - name: List deps/srsc/ntf-core + run: ls -lah ${{ github.workspace }}/deps/srsc/ntf-core + - name: List deps/_build + run: ls -lah ${{ github.workspace }}/deps/_build From 9112acf38512194db9032698fa42815a92ac837c Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 11:22:48 +0300 Subject: [PATCH 008/105] build.yaml: debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/build.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 36990decd9..a49a361706 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -74,9 +74,9 @@ jobs: key: deps-${{ steps.get-hash.outputs.deps_hash }} - name: List deps run: ls -lah ${{ github.workspace }}/deps - - name: List deps/srsc/bde - run: ls -lah ${{ github.workspace }}/deps/srsc/bde - - name: List deps/srsc/ntf-core - run: ls -lah ${{ github.workspace }}/deps/srsc/ntf-core + - name: List deps/srcs/bde + run: ls -lah ${{ github.workspace }}/deps/srcs/bde + - name: List deps/srcs/ntf-core + run: ls -lah ${{ github.workspace }}/deps/srcs/ntf-core - name: List deps/_build run: ls -lah ${{ github.workspace }}/deps/_build From b2b7096ad78a4bcb7191f4a1a5e794c191723e1a Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 11:25:30 +0300 Subject: [PATCH 009/105] build.yaml: debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/build.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a49a361706..b69dd0fc5c 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -80,3 +80,6 @@ jobs: run: ls -lah ${{ github.workspace }}/deps/srcs/ntf-core - name: List deps/_build run: ls -lah ${{ github.workspace }}/deps/_build + - name: List deps/_build/unix-linux-x86_64-6.5.0-gcc-11.4.0-opt_64_cpp17 + run: ls -lah ${{ github.workspace }}/deps/_build/unix-linux-x86_64-6.5.0-gcc-11.4.0-opt_64_cpp17 + \ No newline at end of file From 7fbaaf5f198555065dea9868035ab0497bba4967 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 11:28:17 +0300 Subject: [PATCH 010/105] build.yaml: debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/build.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b69dd0fc5c..56aae31e86 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -74,6 +74,8 @@ jobs: key: deps-${{ steps.get-hash.outputs.deps_hash }} - name: List deps run: ls -lah ${{ github.workspace }}/deps + - name: List deps/srcs + run: ls -lah ${{ github.workspace }}/deps/srcs - name: List deps/srcs/bde run: ls -lah ${{ github.workspace }}/deps/srcs/bde - name: List deps/srcs/ntf-core From 535d4e3c43d920375d4f27d34f206e190ed6e0a9 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 12:48:53 +0300 Subject: [PATCH 011/105] Add reusable workflow sanitizers.yaml Signed-off-by: Aleksandr Ivanov --- .github/workflows/build.yaml | 49 ++++++++++++++++++------------- .github/workflows/sanitizers.yaml | 35 ++++++++++++++++++++++ 2 files changed, 63 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/sanitizers.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 56aae31e86..d239f0669e 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -60,28 +60,35 @@ jobs: key: deps-${{ steps.get-hash.outputs.deps_hash }} build_and_run_asan: - name: Build and run AddressSanitizer - runs-on: ubuntu-latest needs: build_dependencies - steps: - - uses: actions/checkout@v4 - - name: Get dependencies hash - id: get-hash - run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT - - uses: actions/cache/restore@v4 - with: - path: deps - key: deps-${{ steps.get-hash.outputs.deps_hash }} + uses: ${{ github.workspace }}/.github/workflows/sanitizers.yaml + with: + sanitizer-name: asan + working-directory: ${{ github.workspace }} + + # build_and_run_asan: + # name: Build and run AddressSanitizer + # runs-on: ubuntu-latest + # needs: build_dependencies + # steps: + # - uses: actions/checkout@v4 + # - name: Get dependencies hash + # id: get-hash + # run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT + # - uses: actions/cache/restore@v4 + # with: + # path: deps + # key: deps-${{ steps.get-hash.outputs.deps_hash }} + # - name: List deps + # run: ls -lah ${{ github.workspace }}/deps + # - name: List deps/srcs + # run: ls -lah ${{ github.workspace }}/deps/srcs + # - name: List deps/srcs/bde + # run: ls -lah ${{ github.workspace }}/deps/srcs/bde + # - name: List deps/srcs/ntf-core + # run: ls -lah ${{ github.workspace }}/deps/srcs/ntf-core + # - name: List deps/_build + # run: ls -lah ${{ github.workspace }}/deps/_build - name: List deps run: ls -lah ${{ github.workspace }}/deps - - name: List deps/srcs - run: ls -lah ${{ github.workspace }}/deps/srcs - - name: List deps/srcs/bde - run: ls -lah ${{ github.workspace }}/deps/srcs/bde - - name: List deps/srcs/ntf-core - run: ls -lah ${{ github.workspace }}/deps/srcs/ntf-core - - name: List deps/_build - run: ls -lah ${{ github.workspace }}/deps/_build - - name: List deps/_build/unix-linux-x86_64-6.5.0-gcc-11.4.0-opt_64_cpp17 - run: ls -lah ${{ github.workspace }}/deps/_build/unix-linux-x86_64-6.5.0-gcc-11.4.0-opt_64_cpp17 \ No newline at end of file diff --git a/.github/workflows/sanitizers.yaml b/.github/workflows/sanitizers.yaml new file mode 100644 index 0000000000..93113620ff --- /dev/null +++ b/.github/workflows/sanitizers.yaml @@ -0,0 +1,35 @@ +name: Sanitizers + +on: + workflow_call: + inputs: + sanitizer-name: + description: Sanitizer name (asan/msan/tsan/ubsan) + type: string + required: true + working-directory: + description: Working directory to use + type: string + default: './' + + build_and_run_sanitizer: + name: Build and run Sanitizer + runs-on: ubuntu-latest + defaults: + run: + working-directory: ${{ inputs.working-directory }} + steps: + - uses: actions/checkout@v4 + - name: Get dependencies hash + id: get-hash + run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT + - uses: actions/cache/restore@v4 + with: + path: deps + key: deps-${{ steps.get-hash.outputs.deps_hash }} + - name: List deps/srcs + run: ls -lah ${{ github.workspace }}/deps/srcs + - name: San name + run: echo ${{ inputs.sanitizer-name }} + - name: ROOT name + run: echo ${{ inputs.working-directory }} From 349cf97d5ef959e1138c101ee491468096e57be7 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 12:51:02 +0300 Subject: [PATCH 012/105] debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index d239f0669e..e57b6093a5 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -89,6 +89,6 @@ jobs: # run: ls -lah ${{ github.workspace }}/deps/srcs/ntf-core # - name: List deps/_build # run: ls -lah ${{ github.workspace }}/deps/_build - - name: List deps - run: ls -lah ${{ github.workspace }}/deps + # - name: List deps + # run: ls -lah ${{ github.workspace }}/deps \ No newline at end of file From 5056ec4f977aadcbb026cd3c5123e007384bb4a7 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 12:57:17 +0300 Subject: [PATCH 013/105] debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e57b6093a5..e664251609 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -63,8 +63,8 @@ jobs: needs: build_dependencies uses: ${{ github.workspace }}/.github/workflows/sanitizers.yaml with: - sanitizer-name: asan - working-directory: ${{ github.workspace }} + sanitizer-name: asan + # working-directory: ${{ github.workspace }} # build_and_run_asan: # name: Build and run AddressSanitizer From 16848c82a196ec54a8a003f2897a3298317e9705 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 12:58:23 +0300 Subject: [PATCH 014/105] debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/sanitizers.yaml b/.github/workflows/sanitizers.yaml index 93113620ff..e1b3641b8a 100644 --- a/.github/workflows/sanitizers.yaml +++ b/.github/workflows/sanitizers.yaml @@ -19,16 +19,16 @@ on: run: working-directory: ${{ inputs.working-directory }} steps: - - uses: actions/checkout@v4 - - name: Get dependencies hash - id: get-hash - run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT - - uses: actions/cache/restore@v4 - with: - path: deps - key: deps-${{ steps.get-hash.outputs.deps_hash }} - - name: List deps/srcs - run: ls -lah ${{ github.workspace }}/deps/srcs + # - uses: actions/checkout@v4 + # - name: Get dependencies hash + # id: get-hash + # run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT + # - uses: actions/cache/restore@v4 + # with: + # path: deps + # key: deps-${{ steps.get-hash.outputs.deps_hash }} + # - name: List deps/srcs + # run: ls -lah ${{ github.workspace }}/deps/srcs - name: San name run: echo ${{ inputs.sanitizer-name }} - name: ROOT name From 57910acc06505502454e4b6ad08b183889a57388 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 12:59:53 +0300 Subject: [PATCH 015/105] debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/build.yaml | 60 ++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e664251609..c5f57091ce 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -59,36 +59,36 @@ jobs: path: deps key: deps-${{ steps.get-hash.outputs.deps_hash }} - build_and_run_asan: - needs: build_dependencies - uses: ${{ github.workspace }}/.github/workflows/sanitizers.yaml - with: - sanitizer-name: asan - # working-directory: ${{ github.workspace }} - # build_and_run_asan: - # name: Build and run AddressSanitizer - # runs-on: ubuntu-latest # needs: build_dependencies - # steps: - # - uses: actions/checkout@v4 - # - name: Get dependencies hash - # id: get-hash - # run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT - # - uses: actions/cache/restore@v4 - # with: - # path: deps - # key: deps-${{ steps.get-hash.outputs.deps_hash }} - # - name: List deps - # run: ls -lah ${{ github.workspace }}/deps - # - name: List deps/srcs - # run: ls -lah ${{ github.workspace }}/deps/srcs - # - name: List deps/srcs/bde - # run: ls -lah ${{ github.workspace }}/deps/srcs/bde - # - name: List deps/srcs/ntf-core - # run: ls -lah ${{ github.workspace }}/deps/srcs/ntf-core - # - name: List deps/_build - # run: ls -lah ${{ github.workspace }}/deps/_build - # - name: List deps - # run: ls -lah ${{ github.workspace }}/deps + # uses: ${{ github.workspace }}/.github/workflows/sanitizers.yaml + # with: + # sanitizer-name: asan + # working-directory: ${{ github.workspace }} + + build_and_run_asan: + name: Build and run AddressSanitizer + runs-on: ubuntu-latest + needs: build_dependencies + steps: + - uses: actions/checkout@v4 + - name: Get dependencies hash + id: get-hash + run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT + - uses: actions/cache/restore@v4 + with: + path: deps + key: deps-${{ steps.get-hash.outputs.deps_hash }} + - name: List deps + run: ls -lah ${{ github.workspace }}/deps + - name: List deps/srcs + run: ls -lah ${{ github.workspace }}/deps/srcs + - name: List deps/srcs/bde + run: ls -lah ${{ github.workspace }}/deps/srcs/bde + - name: List deps/srcs/ntf-core + run: ls -lah ${{ github.workspace }}/deps/srcs/ntf-core + - name: List deps/_build + run: ls -lah ${{ github.workspace }}/deps/_build + - name: List deps + run: ls -lah ${{ github.workspace }}/deps \ No newline at end of file From 52ff967b587652f042de8630c6277d489144d608 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 13:06:20 +0300 Subject: [PATCH 016/105] debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/build.yaml | 60 +++++++++++++++---------------- .github/workflows/sanitizers.yaml | 51 +++++++++++++------------- 2 files changed, 56 insertions(+), 55 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c5f57091ce..1d2a82f96c 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -59,36 +59,36 @@ jobs: path: deps key: deps-${{ steps.get-hash.outputs.deps_hash }} - # build_and_run_asan: - # needs: build_dependencies - # uses: ${{ github.workspace }}/.github/workflows/sanitizers.yaml - # with: - # sanitizer-name: asan + build_and_run_asan: + # needs: build_dependencies + uses: ${{ github.workspace }}/.github/workflows/sanitizers.yaml + with: + sanitizer-name: 'asan' # working-directory: ${{ github.workspace }} - build_and_run_asan: - name: Build and run AddressSanitizer - runs-on: ubuntu-latest - needs: build_dependencies - steps: - - uses: actions/checkout@v4 - - name: Get dependencies hash - id: get-hash - run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT - - uses: actions/cache/restore@v4 - with: - path: deps - key: deps-${{ steps.get-hash.outputs.deps_hash }} - - name: List deps - run: ls -lah ${{ github.workspace }}/deps - - name: List deps/srcs - run: ls -lah ${{ github.workspace }}/deps/srcs - - name: List deps/srcs/bde - run: ls -lah ${{ github.workspace }}/deps/srcs/bde - - name: List deps/srcs/ntf-core - run: ls -lah ${{ github.workspace }}/deps/srcs/ntf-core - - name: List deps/_build - run: ls -lah ${{ github.workspace }}/deps/_build - - name: List deps - run: ls -lah ${{ github.workspace }}/deps + # build_and_run_asan: + # name: Build and run AddressSanitizer + # runs-on: ubuntu-latest + # needs: build_dependencies + # steps: + # - uses: actions/checkout@v4 + # - name: Get dependencies hash + # id: get-hash + # run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT + # - uses: actions/cache/restore@v4 + # with: + # path: deps + # key: deps-${{ steps.get-hash.outputs.deps_hash }} + # - name: List deps + # run: ls -lah ${{ github.workspace }}/deps + # - name: List deps/srcs + # run: ls -lah ${{ github.workspace }}/deps/srcs + # - name: List deps/srcs/bde + # run: ls -lah ${{ github.workspace }}/deps/srcs/bde + # - name: List deps/srcs/ntf-core + # run: ls -lah ${{ github.workspace }}/deps/srcs/ntf-core + # - name: List deps/_build + # run: ls -lah ${{ github.workspace }}/deps/_build + # - name: List deps + # run: ls -lah ${{ github.workspace }}/deps \ No newline at end of file diff --git a/.github/workflows/sanitizers.yaml b/.github/workflows/sanitizers.yaml index e1b3641b8a..4d45f97d76 100644 --- a/.github/workflows/sanitizers.yaml +++ b/.github/workflows/sanitizers.yaml @@ -7,29 +7,30 @@ on: description: Sanitizer name (asan/msan/tsan/ubsan) type: string required: true - working-directory: - description: Working directory to use - type: string - default: './' + # working-directory: + # description: Working directory to use + # type: string + # default: './' - build_and_run_sanitizer: - name: Build and run Sanitizer - runs-on: ubuntu-latest - defaults: - run: - working-directory: ${{ inputs.working-directory }} - steps: - # - uses: actions/checkout@v4 - # - name: Get dependencies hash - # id: get-hash - # run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT - # - uses: actions/cache/restore@v4 - # with: - # path: deps - # key: deps-${{ steps.get-hash.outputs.deps_hash }} - # - name: List deps/srcs - # run: ls -lah ${{ github.workspace }}/deps/srcs - - name: San name - run: echo ${{ inputs.sanitizer-name }} - - name: ROOT name - run: echo ${{ inputs.working-directory }} + jobs: + build_and_run_sanitizer: + name: Build and run Sanitizer + runs-on: ubuntu-latest + # defaults: + # run: + # working-directory: ${{ inputs.working-directory }} + steps: + # - uses: actions/checkout@v4 + # - name: Get dependencies hash + # id: get-hash + # run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT + # - uses: actions/cache/restore@v4 + # with: + # path: deps + # key: deps-${{ steps.get-hash.outputs.deps_hash }} + # - name: List deps/srcs + # run: ls -lah ${{ github.workspace }}/deps/srcs + - name: San name + run: echo ${{ inputs.sanitizer-name }} + - name: ROOT name + run: echo ${{ inputs.working-directory }} From c37d1f7722d1f7a2b17a78ecf7b00c8b5da05164 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 13:09:21 +0300 Subject: [PATCH 017/105] debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/build.yaml | 4 +-- .github/workflows/sanitizers.yaml | 44 +++++++++++++++---------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 1d2a82f96c..87f74dc3b4 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -61,9 +61,9 @@ jobs: build_and_run_asan: # needs: build_dependencies - uses: ${{ github.workspace }}/.github/workflows/sanitizers.yaml + uses: ./.github/workflows/sanitizers.yaml with: - sanitizer-name: 'asan' + sanitizer-name: 'asan' # working-directory: ${{ github.workspace }} # build_and_run_asan: diff --git a/.github/workflows/sanitizers.yaml b/.github/workflows/sanitizers.yaml index 4d45f97d76..cac9095aec 100644 --- a/.github/workflows/sanitizers.yaml +++ b/.github/workflows/sanitizers.yaml @@ -12,25 +12,25 @@ on: # type: string # default: './' - jobs: - build_and_run_sanitizer: - name: Build and run Sanitizer - runs-on: ubuntu-latest - # defaults: - # run: - # working-directory: ${{ inputs.working-directory }} - steps: - # - uses: actions/checkout@v4 - # - name: Get dependencies hash - # id: get-hash - # run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT - # - uses: actions/cache/restore@v4 - # with: - # path: deps - # key: deps-${{ steps.get-hash.outputs.deps_hash }} - # - name: List deps/srcs - # run: ls -lah ${{ github.workspace }}/deps/srcs - - name: San name - run: echo ${{ inputs.sanitizer-name }} - - name: ROOT name - run: echo ${{ inputs.working-directory }} +jobs: + build_and_run_sanitizer: + name: Build and run Sanitizer + runs-on: ubuntu-latest + # defaults: + # run: + # working-directory: ${{ inputs.working-directory }} + steps: + # - uses: actions/checkout@v4 + # - name: Get dependencies hash + # id: get-hash + # run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT + # - uses: actions/cache/restore@v4 + # with: + # path: deps + # key: deps-${{ steps.get-hash.outputs.deps_hash }} + # - name: List deps/srcs + # run: ls -lah ${{ github.workspace }}/deps/srcs + - name: San name + run: echo ${{ inputs.sanitizer-name }} + - name: ROOT name + run: echo ${{ inputs.working-directory }} From f850100135e249275c4bfaa3e29685bf44a02580 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 13:17:25 +0300 Subject: [PATCH 018/105] debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/sanitizers.yaml b/.github/workflows/sanitizers.yaml index cac9095aec..cb88cb664e 100644 --- a/.github/workflows/sanitizers.yaml +++ b/.github/workflows/sanitizers.yaml @@ -7,18 +7,18 @@ on: description: Sanitizer name (asan/msan/tsan/ubsan) type: string required: true - # working-directory: - # description: Working directory to use - # type: string - # default: './' + working-directory: + description: Working directory to use + type: string + default: './' jobs: build_and_run_sanitizer: name: Build and run Sanitizer runs-on: ubuntu-latest - # defaults: - # run: - # working-directory: ${{ inputs.working-directory }} + defaults: + run: + working-directory: ${{ inputs.working-directory }} steps: # - uses: actions/checkout@v4 # - name: Get dependencies hash @@ -33,4 +33,4 @@ jobs: - name: San name run: echo ${{ inputs.sanitizer-name }} - name: ROOT name - run: echo ${{ inputs.working-directory }} + run: pwd From b91a7e09f25e882757140439de53f69ae0766e73 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 13:30:45 +0300 Subject: [PATCH 019/105] debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/build.yaml | 2 +- .github/workflows/sanitizers.yaml | 36 +++++++++++++++---------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 87f74dc3b4..a875d6f7e1 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -60,7 +60,7 @@ jobs: key: deps-${{ steps.get-hash.outputs.deps_hash }} build_and_run_asan: - # needs: build_dependencies + needs: build_dependencies uses: ./.github/workflows/sanitizers.yaml with: sanitizer-name: 'asan' diff --git a/.github/workflows/sanitizers.yaml b/.github/workflows/sanitizers.yaml index cb88cb664e..5590aaed9a 100644 --- a/.github/workflows/sanitizers.yaml +++ b/.github/workflows/sanitizers.yaml @@ -7,29 +7,29 @@ on: description: Sanitizer name (asan/msan/tsan/ubsan) type: string required: true - working-directory: - description: Working directory to use - type: string - default: './' + # working-directory: + # description: Working directory to use + # type: string + # default: './' jobs: build_and_run_sanitizer: - name: Build and run Sanitizer + name: Build and run Sanitizer: ${{ inputs.sanitizer-name }} runs-on: ubuntu-latest - defaults: - run: - working-directory: ${{ inputs.working-directory }} + # defaults: + # run: + # working-directory: ${{ inputs.working-directory }} steps: - # - uses: actions/checkout@v4 - # - name: Get dependencies hash - # id: get-hash - # run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT - # - uses: actions/cache/restore@v4 - # with: - # path: deps - # key: deps-${{ steps.get-hash.outputs.deps_hash }} - # - name: List deps/srcs - # run: ls -lah ${{ github.workspace }}/deps/srcs + - uses: actions/checkout@v4 + - name: Get dependencies hash + id: get-hash + run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT + - uses: actions/cache/restore@v4 + with: + path: deps + key: deps-${{ steps.get-hash.outputs.deps_hash }} + - name: List deps/srcs + run: ls -lah ${{ github.workspace }}/deps/srcs - name: San name run: echo ${{ inputs.sanitizer-name }} - name: ROOT name From a2206a436d4d0173abad6d64221c795a319b501e Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 13:31:54 +0300 Subject: [PATCH 020/105] debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/sanitizers.yaml b/.github/workflows/sanitizers.yaml index 5590aaed9a..e15dc0ca39 100644 --- a/.github/workflows/sanitizers.yaml +++ b/.github/workflows/sanitizers.yaml @@ -20,16 +20,16 @@ jobs: # run: # working-directory: ${{ inputs.working-directory }} steps: - - uses: actions/checkout@v4 - - name: Get dependencies hash - id: get-hash - run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT - - uses: actions/cache/restore@v4 - with: - path: deps - key: deps-${{ steps.get-hash.outputs.deps_hash }} - - name: List deps/srcs - run: ls -lah ${{ github.workspace }}/deps/srcs + # - uses: actions/checkout@v4 + # - name: Get dependencies hash + # id: get-hash + # run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT + # - uses: actions/cache/restore@v4 + # with: + # path: deps + # key: deps-${{ steps.get-hash.outputs.deps_hash }} + # - name: List deps/srcs + # run: ls -lah ${{ github.workspace }}/deps/srcs - name: San name run: echo ${{ inputs.sanitizer-name }} - name: ROOT name From 37670ed3fba986aaec233581d3eeb8a1cd488058 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 13:32:47 +0300 Subject: [PATCH 021/105] debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a875d6f7e1..87f74dc3b4 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -60,7 +60,7 @@ jobs: key: deps-${{ steps.get-hash.outputs.deps_hash }} build_and_run_asan: - needs: build_dependencies + # needs: build_dependencies uses: ./.github/workflows/sanitizers.yaml with: sanitizer-name: 'asan' From 32ee569a35f86acdfcdfc8b93eb2ec9a189f9ca0 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 13:34:06 +0300 Subject: [PATCH 022/105] debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sanitizers.yaml b/.github/workflows/sanitizers.yaml index e15dc0ca39..99b7ef2455 100644 --- a/.github/workflows/sanitizers.yaml +++ b/.github/workflows/sanitizers.yaml @@ -14,7 +14,7 @@ on: jobs: build_and_run_sanitizer: - name: Build and run Sanitizer: ${{ inputs.sanitizer-name }} + name: Build and run Sanitizer runs-on: ubuntu-latest # defaults: # run: From bd36cb4ac8953c8e5797993ef18e267054571f6b Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 13:36:10 +0300 Subject: [PATCH 023/105] debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/build.yaml | 2 +- .github/workflows/sanitizers.yaml | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 87f74dc3b4..a875d6f7e1 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -60,7 +60,7 @@ jobs: key: deps-${{ steps.get-hash.outputs.deps_hash }} build_and_run_asan: - # needs: build_dependencies + needs: build_dependencies uses: ./.github/workflows/sanitizers.yaml with: sanitizer-name: 'asan' diff --git a/.github/workflows/sanitizers.yaml b/.github/workflows/sanitizers.yaml index 99b7ef2455..65bfd17fe5 100644 --- a/.github/workflows/sanitizers.yaml +++ b/.github/workflows/sanitizers.yaml @@ -20,16 +20,16 @@ jobs: # run: # working-directory: ${{ inputs.working-directory }} steps: - # - uses: actions/checkout@v4 - # - name: Get dependencies hash - # id: get-hash - # run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT - # - uses: actions/cache/restore@v4 - # with: - # path: deps - # key: deps-${{ steps.get-hash.outputs.deps_hash }} - # - name: List deps/srcs - # run: ls -lah ${{ github.workspace }}/deps/srcs + - uses: actions/checkout@v4 + - name: Get dependencies hash + id: get-hash + run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT + - uses: actions/cache/restore@v4 + with: + path: deps + key: deps-${{ steps.get-hash.outputs.deps_hash }} + - name: List deps/srcs + run: ls -lah ${{ github.workspace }}/deps/srcs - name: San name run: echo ${{ inputs.sanitizer-name }} - name: ROOT name From 4a6d2e4e9f6827cee0eb5aeb7d14b8ccf227e0e3 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 13:50:09 +0300 Subject: [PATCH 024/105] debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers.yaml | 10 +++--- .../workflows/sanitizers/build_sanitizer.sh | 35 +++++++++++++++++++ 2 files changed, 41 insertions(+), 4 deletions(-) create mode 100755 .github/workflows/sanitizers/build_sanitizer.sh diff --git a/.github/workflows/sanitizers.yaml b/.github/workflows/sanitizers.yaml index 65bfd17fe5..363f97a99c 100644 --- a/.github/workflows/sanitizers.yaml +++ b/.github/workflows/sanitizers.yaml @@ -30,7 +30,9 @@ jobs: key: deps-${{ steps.get-hash.outputs.deps_hash }} - name: List deps/srcs run: ls -lah ${{ github.workspace }}/deps/srcs - - name: San name - run: echo ${{ inputs.sanitizer-name }} - - name: ROOT name - run: pwd + # - name: San name + # run: echo ${{ inputs.sanitizer-name }} + # - name: ROOT name + # run: pwd + - name: Build BlazingMQ and dependencies with sanitizer instrumentation + run: ./.github/workflows/build_sanitizer.sh diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh new file mode 100755 index 0000000000..7d27dc38f2 --- /dev/null +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# This script can be used to build BlazingMQ, and all of its transitive +# dependencies (up to and including the standard library) using: +# - Clang +# - LLVM libc++ standard library +# - A CMake toolchain file specific for instrumented build +# It is currently used to build instrumented BlazingMQ binaries for CI for all +# Clang sanitizers (i.e. Address/Leak, Memory, Thread, UndefinedBehavior). +# +# It performs the following: +# 1) Install clang compiler. +# 2) Download llvm-project required for libc++ instrumentation. +# 3) Download external dependencies required for instrumentation. +# 4) Build libc++ with the instrumentation specified by . +# 5) Build sanitizer-instrumented dependencies including BDE, NTF, GoogleTest, +# Google Benchmark and zlib. +# 6) Build sanitizer-instrumented BlazingMQ unit tests. +# 7) Generate scripts to run unit tests: +# ./cmake.bld/Linux/run-unittests.sh +# This script is used as-is by CI to run unit tests with sanitizer. + +set -eux + +# :: Required arguments ::::::::::::::::::::::::::::::::::::::::::::::::::::::: +if [ -z ${1} ]; then + echo 'Error: Missing sanitizer name.' >&2 + echo ' (Usage: build_sanitizer.sh )' >&2 + exit 1 +fi + +SANITIZER_NAME="${1}" + +echo SANITIZER_NAME: "${SANITIZER_NAME}" +echo ROOT: "${PWD}" From 623bd42cd65e1ef98a214457778bb59104b7ecaa Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 13:51:13 +0300 Subject: [PATCH 025/105] debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sanitizers.yaml b/.github/workflows/sanitizers.yaml index 363f97a99c..bbb4d98892 100644 --- a/.github/workflows/sanitizers.yaml +++ b/.github/workflows/sanitizers.yaml @@ -35,4 +35,4 @@ jobs: # - name: ROOT name # run: pwd - name: Build BlazingMQ and dependencies with sanitizer instrumentation - run: ./.github/workflows/build_sanitizer.sh + run: ./.github/workflows/sanitizers/build_sanitizer.sh From 532bedf0872181ca97ebf543237d9a0126049d3d Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 13:52:34 +0300 Subject: [PATCH 026/105] debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sanitizers.yaml b/.github/workflows/sanitizers.yaml index bbb4d98892..e26cd045f9 100644 --- a/.github/workflows/sanitizers.yaml +++ b/.github/workflows/sanitizers.yaml @@ -35,4 +35,4 @@ jobs: # - name: ROOT name # run: pwd - name: Build BlazingMQ and dependencies with sanitizer instrumentation - run: ./.github/workflows/sanitizers/build_sanitizer.sh + run: ./.github/workflows/sanitizers/build_sanitizer.sh ${{ inputs.sanitizer-name }} From db59b0a8558ce12922bfcdbc67c36aefa4510b56 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 15:03:16 +0300 Subject: [PATCH 027/105] debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/build_sanitizer.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 7d27dc38f2..610827b455 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -33,3 +33,8 @@ SANITIZER_NAME="${1}" echo SANITIZER_NAME: "${SANITIZER_NAME}" echo ROOT: "${PWD}" + +# Install prerequisites +apt update && apt install -y lsb-release wget software-properties-common gnupg git curl jq ninja-build bison libfl-dev pkg-config cmake && rm -rf /var/lib/apt/lists/* /var/log/dpkg.log + +echo cmake --version From d31d6df86ad6fa35539a9f9f9282c003f69ebf2f Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 15:10:59 +0300 Subject: [PATCH 028/105] debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/build_sanitizer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 610827b455..4c6225fd54 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -35,6 +35,6 @@ echo SANITIZER_NAME: "${SANITIZER_NAME}" echo ROOT: "${PWD}" # Install prerequisites -apt update && apt install -y lsb-release wget software-properties-common gnupg git curl jq ninja-build bison libfl-dev pkg-config cmake && rm -rf /var/lib/apt/lists/* /var/log/dpkg.log +sudo apt-get update && sudo apt-get install -qy lsb-release wget software-properties-common gnupg git cmake curl jq ninja-build bison libfl-dev pkg-config echo cmake --version From 4a6e4cf413b2a700637e5824ecd1d77d8bc6167c Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 15:30:58 +0300 Subject: [PATCH 029/105] debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/build_sanitizer.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 4c6225fd54..4695ab05fb 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -35,6 +35,11 @@ echo SANITIZER_NAME: "${SANITIZER_NAME}" echo ROOT: "${PWD}" # Install prerequisites -sudo apt-get update && sudo apt-get install -qy lsb-release wget software-properties-common gnupg git cmake curl jq ninja-build bison libfl-dev pkg-config +sudo apt-get update && sudo apt-get install -qy lsb-release wget software-properties-common gnupg git curl jq ninja-build bison libfl-dev pkg-config -echo cmake --version +# Prerequisites for LLVM installation: latest cmake version, Ubuntu apt repository contains cmake version 3.22.1 +wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | sudo tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null +apt-add-repository -y "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" +sudo apt-get update && sudo apt-get install -qy cmake + +cmake --version From 12f8cbeb4c7399e82e7971d1626135ff31331c07 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 15:36:06 +0300 Subject: [PATCH 030/105] debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/build_sanitizer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 4695ab05fb..def78217c7 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -39,7 +39,7 @@ sudo apt-get update && sudo apt-get install -qy lsb-release wget software-proper # Prerequisites for LLVM installation: latest cmake version, Ubuntu apt repository contains cmake version 3.22.1 wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | sudo tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null -apt-add-repository -y "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" +sudo apt-add-repository -y "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" sudo apt-get update && sudo apt-get install -qy cmake cmake --version From efacf0cdbac8d38a500defd37dc2519da1910bd7 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 15:38:07 +0300 Subject: [PATCH 031/105] debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/build_sanitizer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index def78217c7..41628b7629 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -40,6 +40,6 @@ sudo apt-get update && sudo apt-get install -qy lsb-release wget software-proper # Prerequisites for LLVM installation: latest cmake version, Ubuntu apt repository contains cmake version 3.22.1 wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | sudo tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null sudo apt-add-repository -y "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" -sudo apt-get update && sudo apt-get install -qy cmake +sudo apt-get install -qy cmake cmake --version From 988e566bb2b376f4b092f533222581ce5a69cf49 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 15:49:33 +0300 Subject: [PATCH 032/105] debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .../workflows/sanitizers/build_sanitizer.sh | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 41628b7629..c2e2c66375 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -42,4 +42,35 @@ wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | sudo apt-add-repository -y "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" sudo apt-get install -qy cmake -cmake --version +# Install LLVM +wget https://apt.llvm.org/llvm.sh +chmod +x llvm.sh +LLVM_VERSION=18 +sudo ./llvm.sh ${LLVM_VERSION} all + +# Create version-agnostic pointers to required LLVM binaries. +ln -sf /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang +ln -sf /usr/bin/clang++-${LLVM_VERSION} /usr/bin/clang++ +ln -sf /usr/bin/llvm-symbolizer-${LLVM_VERSION} /usr/bin/llvm-symbolizer + +# Parse sanitizers config +cfgquery() { + jq "${1}" "./.github/workflows/sanitizers.json" --raw-output +} +LLVM_SANITIZER_NAME="$(cfgquery .${SANITIZER_NAME}.llvm_sanitizer_name)" +# Check if llvm specific cmake options are present for the given sanitizer +LLVM_SPECIFIC_CMAKE_OPTIONS="$(cfgquery .${SANITIZER_NAME}.llvm_specific_cmake_options)" +if [[ "$LLVM_SPECIFIC_CMAKE_OPTIONS" == null ]]; then LLVM_SPECIFIC_CMAKE_OPTIONS=""; fi + +# Set some initial constants +PARALLELISM=8 + +# DIR_ROOT="${PWD}" +# DIR_EXTERNAL="${DIR_ROOT}/_external" +# DIR_SRCS_EXT="${DIR_EXTERNAL}/srcs" +# DIR_BUILD_EXT="${DIR_SRCS_EXT}/cmake.bld" + +# DIR_SRC_BMQ="${DIR_SRCS_EXT}/blazingmq" +# DIR_BUILD_BMQ="${DIR_SRC_BMQ}/cmake.bld/Linux" + +echo LLVM_SANITIZER_NAME: ${LLVM_SANITIZER_NAME} From dbfcfcc43bac67742a8612c17e16b3b1c479ec4e Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 15:52:58 +0300 Subject: [PATCH 033/105] debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/build_sanitizer.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index c2e2c66375..0c58902790 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -49,9 +49,9 @@ LLVM_VERSION=18 sudo ./llvm.sh ${LLVM_VERSION} all # Create version-agnostic pointers to required LLVM binaries. -ln -sf /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang -ln -sf /usr/bin/clang++-${LLVM_VERSION} /usr/bin/clang++ -ln -sf /usr/bin/llvm-symbolizer-${LLVM_VERSION} /usr/bin/llvm-symbolizer +sudo ln -sf /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang +sudo ln -sf /usr/bin/clang++-${LLVM_VERSION} /usr/bin/clang++ +sudo ln -sf /usr/bin/llvm-symbolizer-${LLVM_VERSION} /usr/bin/llvm-symbolizer # Parse sanitizers config cfgquery() { From f077610494b28a4dca6b9c89706c6c0a9353a0fe Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 15:56:14 +0300 Subject: [PATCH 034/105] debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/build_sanitizer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 0c58902790..1fa55698d2 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -55,7 +55,7 @@ sudo ln -sf /usr/bin/llvm-symbolizer-${LLVM_VERSION} /usr/bin/llvm-symbolizer # Parse sanitizers config cfgquery() { - jq "${1}" "./.github/workflows/sanitizers.json" --raw-output + jq "${1}" "./.github/workflows/sanitizers/sanitizers.json" --raw-output } LLVM_SANITIZER_NAME="$(cfgquery .${SANITIZER_NAME}.llvm_sanitizer_name)" # Check if llvm specific cmake options are present for the given sanitizer From 64673a87c1d6368af4479a6ef40f529c6d909853 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 16:09:31 +0300 Subject: [PATCH 035/105] debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .../workflows/sanitizers/build_sanitizer.sh | 44 +++++++++++++++---- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 1fa55698d2..eb2d78b4b4 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -38,7 +38,7 @@ echo ROOT: "${PWD}" sudo apt-get update && sudo apt-get install -qy lsb-release wget software-properties-common gnupg git curl jq ninja-build bison libfl-dev pkg-config # Prerequisites for LLVM installation: latest cmake version, Ubuntu apt repository contains cmake version 3.22.1 -wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | sudo tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null +# wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | sudo tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null sudo apt-add-repository -y "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" sudo apt-get install -qy cmake @@ -65,12 +65,40 @@ if [[ "$LLVM_SPECIFIC_CMAKE_OPTIONS" == null ]]; then LLVM_SPECIFIC_CMAKE_OPTION # Set some initial constants PARALLELISM=8 -# DIR_ROOT="${PWD}" -# DIR_EXTERNAL="${DIR_ROOT}/_external" -# DIR_SRCS_EXT="${DIR_EXTERNAL}/srcs" -# DIR_BUILD_EXT="${DIR_SRCS_EXT}/cmake.bld" +DIR_ROOT="${PWD}" +DIR_EXTERNAL="${DIR_ROOT}/deps" +DIR_SRCS_EXT="${DIR_EXTERNAL}/srcs" +DIR_BUILD_EXT="${DIR_SRCS_EXT}/cmake.bld" -# DIR_SRC_BMQ="${DIR_SRCS_EXT}/blazingmq" -# DIR_BUILD_BMQ="${DIR_SRC_BMQ}/cmake.bld/Linux" +# :: checkoutGitRepo() subroutine ::::::::::::::::::::::::::::::::::::::::::::: +checkoutGitRepo() { + local repo=$1 + local ref=$2 + local repoDir=$3 + echo "Checking out ${repo} at ${ref}" -echo LLVM_SANITIZER_NAME: ${LLVM_SANITIZER_NAME} + local repoPath="${DIR_SRCS_EXT}/${repoDir}" + + git clone -b ${ref} ${repo} \ + --depth 1 --single-branch --no-tags -c advice.detachedHead=false "${repoPath}" +} +github_url() { echo "https://github.com/$1.git"; } + +# :: Download external dependencies ::::::::::::::::::::::::::::::: +mkdir -p ${DIR_SRCS_EXT} + +# Download LLVM +LLVM_TAG="llvmorg-18.1.8" +checkoutGitRepo "$(github_url llvm/llvm-project)" "${LLVM_TAG}" "llvm-project" + +# Download google-benchmark +GOOGLE_BENCHMARK_TAG="v1.8.4" +checkoutGitRepo "$(github_url google/benchmark)" "${GOOGLE_BENCHMARK_TAG}" "google-benchmark" + +# Download googletest +GOOGLETEST_TAG="v1.14.0" +checkoutGitRepo "$(github_url google/googletest)" "${GOOGLETEST_TAG}" "googletest" + +# Download zlib +ZLIB_TAG="v1.3.1" +checkoutGitRepo "$(github_url madler/zlib)" "${ZLIB_TAG}" "zlib" From bdb4f690bd36808b4023f8ff28e1238cbe76a313 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 16:16:36 +0300 Subject: [PATCH 036/105] debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .../workflows/sanitizers/build_sanitizer.sh | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index eb2d78b4b4..86acdae770 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -37,8 +37,8 @@ echo ROOT: "${PWD}" # Install prerequisites sudo apt-get update && sudo apt-get install -qy lsb-release wget software-properties-common gnupg git curl jq ninja-build bison libfl-dev pkg-config -# Prerequisites for LLVM installation: latest cmake version, Ubuntu apt repository contains cmake version 3.22.1 -# wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | sudo tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null +# Install prerequisites for LLVM: latest cmake version, Ubuntu apt repository contains cmake version 3.22.1 +wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | sudo tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null sudo apt-add-repository -y "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" sudo apt-get install -qy cmake @@ -102,3 +102,28 @@ checkoutGitRepo "$(github_url google/googletest)" "${GOOGLETEST_TAG}" "googletes # Download zlib ZLIB_TAG="v1.3.1" checkoutGitRepo "$(github_url madler/zlib)" "${ZLIB_TAG}" "zlib" + +# :: Build libc++ with required instrumentation ::::::::::::::::::::::::::::::: +# +# The extent to which all dependencies to be compiled with sanitizer-support +# varies by sanitizer. MemorySanitizer is especially unforgiving: Failing to +# link against an instrumented standard library will yield many false +# positives. Concensus is that compiling libc++ with `-fsanitize=memory` is a +# significantly easier endeavor than doing the same with libstdc++ (the gcc +# standard library). +# +# We therefore opt to use libc++ here, just to ensure maximum flexibility. We +# follow build instructions from https://libcxx.llvm.org/BuildingLibcxx.html +LIBCXX_SRC_PATH="${DIR_SRCS_EXT}/llvm-project/runtimes" +LIBCXX_BUILD_PATH="${LIBCXX_SRC_PATH}/cmake.bld" + +cmake -B "${LIBCXX_BUILD_PATH}" \ + -S "${LIBCXX_SRC_PATH}" \ + -DCMAKE_BUILD_TYPE="Debug" \ + -DCMAKE_C_COMPILER="clang" \ + -DCMAKE_CXX_COMPILER="clang++" \ + -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ + -DLLVM_USE_SANITIZER="${LLVM_SANITIZER_NAME}" \ + ${LLVM_SPECIFIC_CMAKE_OPTIONS} + +cmake --build "${LIBCXX_BUILD_PATH}" -j${PARALLELISM} --target cxx cxxabi unwind generate-cxx-headers From 04b7fb6990e4209d12b1d7aa6abb725ff81106ec Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 16:44:10 +0300 Subject: [PATCH 037/105] debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .../workflows/sanitizers/build_sanitizer.sh | 47 ++++++++++++++++++- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 86acdae770..8d3e458a84 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -55,7 +55,7 @@ sudo ln -sf /usr/bin/llvm-symbolizer-${LLVM_VERSION} /usr/bin/llvm-symbolizer # Parse sanitizers config cfgquery() { - jq "${1}" "./.github/workflows/sanitizers/sanitizers.json" --raw-output + jq "${1}" "${DIR_SCRIPTS}/sanitizers.json" --raw-output } LLVM_SANITIZER_NAME="$(cfgquery .${SANITIZER_NAME}.llvm_sanitizer_name)" # Check if llvm specific cmake options are present for the given sanitizer @@ -66,9 +66,13 @@ if [[ "$LLVM_SPECIFIC_CMAKE_OPTIONS" == null ]]; then LLVM_SPECIFIC_CMAKE_OPTION PARALLELISM=8 DIR_ROOT="${PWD}" +DIR_SCRIPTS="${DIR_ROOT}/.github/workflows/sanitizers" DIR_EXTERNAL="${DIR_ROOT}/deps" DIR_SRCS_EXT="${DIR_EXTERNAL}/srcs" -DIR_BUILD_EXT="${DIR_SRCS_EXT}/cmake.bld" +DIR_BUILD_EXT="${DIR_EXTERNAL}/cmake.bld" + +DIR_SRC_BMQ="${DIR_ROOT}" +DIR_BUILD_BMQ="${DIR_SRC_BMQ}/cmake.bld/Linux" # :: checkoutGitRepo() subroutine ::::::::::::::::::::::::::::::::::::::::::::: checkoutGitRepo() { @@ -127,3 +131,42 @@ cmake -B "${LIBCXX_BUILD_PATH}" \ ${LLVM_SPECIFIC_CMAKE_OPTIONS} cmake --build "${LIBCXX_BUILD_PATH}" -j${PARALLELISM} --target cxx cxxabi unwind generate-cxx-headers + +# Variables read by our custom CMake toolchain used to build everything else. +export LIBCXX_BUILD_PATH="$(realpath ${LIBCXX_BUILD_PATH})" +export DIR_SRC_BMQ="${DIR_SRC_BMQ}" + +# :: Build BDE + NTF :::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +TOOLCHAIN_PATH="${DIR_SCRIPTS}/clang-libcxx-${SANITIZER_NAME}.cmake" +export CC="clang" +export CXX="clang++" +export BBS_BUILD_SYSTEM="ON" +PATH="$PATH:$(realpath ${DIR_SRCS_EXT}/bde-tools/bin)" +export PATH + +pushd ${DIR_SRCS_EXT} + +pushd "bde" +eval "$(bbs_build_env -u dbg_64_safe_cpp20 -b "${DIR_BUILD_EXT}/bde")" +bbs_build configure --toolchain "${TOOLCHAIN_PATH}" +bbs_build build -j${PARALLELISM} +bbs_build --install=/opt/bb --prefix=/ install +popd + +pushd "ntf-core" +# TODO The deprecated flag "-fcoroutines-ts" has been removed in clang +# 17.0.1, but NTF is still using it. We manually change this flag until +# the fix in issue 175307231 is resolved. +sed -i 's/fcoroutines-ts/fcoroutines/g' 'repository.cmake' + +./configure --keep \ + --prefix /opt/bb \ + --output "${DIR_BUILD_EXT}/ntf" \ + --without-warnings-as-errors \ + --without-usage-examples \ + --without-applications \ + --ufid 'dbg_64_safe_cpp20' \ + --toolchain "${TOOLCHAIN_PATH}" +make -j${PARALLELISM} +make install +popd From f8d7278e2ac940f290a8e97f82e7318cf5120101 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 16:49:18 +0300 Subject: [PATCH 038/105] debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .../workflows/sanitizers/build_sanitizer.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 8d3e458a84..97101f16ea 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -53,15 +53,6 @@ sudo ln -sf /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang sudo ln -sf /usr/bin/clang++-${LLVM_VERSION} /usr/bin/clang++ sudo ln -sf /usr/bin/llvm-symbolizer-${LLVM_VERSION} /usr/bin/llvm-symbolizer -# Parse sanitizers config -cfgquery() { - jq "${1}" "${DIR_SCRIPTS}/sanitizers.json" --raw-output -} -LLVM_SANITIZER_NAME="$(cfgquery .${SANITIZER_NAME}.llvm_sanitizer_name)" -# Check if llvm specific cmake options are present for the given sanitizer -LLVM_SPECIFIC_CMAKE_OPTIONS="$(cfgquery .${SANITIZER_NAME}.llvm_specific_cmake_options)" -if [[ "$LLVM_SPECIFIC_CMAKE_OPTIONS" == null ]]; then LLVM_SPECIFIC_CMAKE_OPTIONS=""; fi - # Set some initial constants PARALLELISM=8 @@ -74,6 +65,15 @@ DIR_BUILD_EXT="${DIR_EXTERNAL}/cmake.bld" DIR_SRC_BMQ="${DIR_ROOT}" DIR_BUILD_BMQ="${DIR_SRC_BMQ}/cmake.bld/Linux" +# Parse sanitizers config +cfgquery() { + jq "${1}" "${DIR_SCRIPTS}/sanitizers.json" --raw-output +} +LLVM_SANITIZER_NAME="$(cfgquery .${SANITIZER_NAME}.llvm_sanitizer_name)" +# Check if llvm specific cmake options are present for the given sanitizer +LLVM_SPECIFIC_CMAKE_OPTIONS="$(cfgquery .${SANITIZER_NAME}.llvm_specific_cmake_options)" +if [[ "$LLVM_SPECIFIC_CMAKE_OPTIONS" == null ]]; then LLVM_SPECIFIC_CMAKE_OPTIONS=""; fi + # :: checkoutGitRepo() subroutine ::::::::::::::::::::::::::::::::::::::::::::: checkoutGitRepo() { local repo=$1 From 4ee909d9452c97783928ba3c8fae27fff1b36a67 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 16:57:11 +0300 Subject: [PATCH 039/105] debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/clang-libcxx-asan.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sanitizers/clang-libcxx-asan.cmake b/.github/workflows/sanitizers/clang-libcxx-asan.cmake index eda60f2267..7bf97294be 100644 --- a/.github/workflows/sanitizers/clang-libcxx-asan.cmake +++ b/.github/workflows/sanitizers/clang-libcxx-asan.cmake @@ -13,7 +13,7 @@ cmake_minimum_required (VERSION 3.25) # endif() # endif() -include("/sanitizers/BBToolchain64.cmake") +# include("/sanitizers/BBToolchain64.cmake") if(DEFINED ENV{CC}) set(CMAKE_C_COMPILER $ENV{CC} CACHE STRING "Instrumentation C compiler" FORCE) From 824a6e586f354ed5348ef244b3015dab1155de26 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 9 Jul 2024 17:05:24 +0300 Subject: [PATCH 040/105] debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/build_sanitizer.sh | 1 + .github/workflows/sanitizers/clang-libcxx-asan.cmake | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 97101f16ea..939be7b32b 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -135,6 +135,7 @@ cmake --build "${LIBCXX_BUILD_PATH}" -j${PARALLELISM} --target cxx cxxabi unwind # Variables read by our custom CMake toolchain used to build everything else. export LIBCXX_BUILD_PATH="$(realpath ${LIBCXX_BUILD_PATH})" export DIR_SRC_BMQ="${DIR_SRC_BMQ}" +export DIR_SCRIPTS="${DIR_SCRIPTS}" # :: Build BDE + NTF :::::::::::::::::::::::::::::::::::::::::::::::::::::::::: TOOLCHAIN_PATH="${DIR_SCRIPTS}/clang-libcxx-${SANITIZER_NAME}.cmake" diff --git a/.github/workflows/sanitizers/clang-libcxx-asan.cmake b/.github/workflows/sanitizers/clang-libcxx-asan.cmake index 7bf97294be..580637eec4 100644 --- a/.github/workflows/sanitizers/clang-libcxx-asan.cmake +++ b/.github/workflows/sanitizers/clang-libcxx-asan.cmake @@ -13,7 +13,7 @@ cmake_minimum_required (VERSION 3.25) # endif() # endif() -# include("/sanitizers/BBToolchain64.cmake") +include("$ENV{DIR_SCRIPTS}/BBToolchain64.cmake") if(DEFINED ENV{CC}) set(CMAKE_C_COMPILER $ENV{CC} CACHE STRING "Instrumentation C compiler" FORCE) From b072e368a9321ce20316d231fa4786ce0f870bb7 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Wed, 10 Jul 2024 11:24:30 +0300 Subject: [PATCH 041/105] build.yaml: debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .../workflows/sanitizers/build_sanitizer.sh | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 939be7b32b..a672e4733a 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -137,6 +137,28 @@ export LIBCXX_BUILD_PATH="$(realpath ${LIBCXX_BUILD_PATH})" export DIR_SRC_BMQ="${DIR_SRC_BMQ}" export DIR_SCRIPTS="${DIR_SCRIPTS}" + +################################################# +echo ################################################# +sudo update-alternatives --all +echo ################################################# + +sudo update-alternatives --remove-all gcc +sudo update-alternatives --remove-all llvm +sudo update-alternatives --remove-all clang + +# sudo ln -sf /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang +# sudo ln -sf /usr/bin/clang++-${LLVM_VERSION} /usr/bin/clang++ +sudo update-alternatives \ + --install /usr/bin/clang clang /usr/bin/clang-18 100 \ + --slave /usr/bin/clang++ clang++ /usr/bin/clang++-18 \ + --slave /usr/bin/lld lld /usr/bin/lld-18 \ + +echo ################################################# +sudo update-alternatives --all +echo ################################################# +################################################# + # :: Build BDE + NTF :::::::::::::::::::::::::::::::::::::::::::::::::::::::::: TOOLCHAIN_PATH="${DIR_SCRIPTS}/clang-libcxx-${SANITIZER_NAME}.cmake" export CC="clang" @@ -150,6 +172,11 @@ pushd ${DIR_SRCS_EXT} pushd "bde" eval "$(bbs_build_env -u dbg_64_safe_cpp20 -b "${DIR_BUILD_EXT}/bde")" bbs_build configure --toolchain "${TOOLCHAIN_PATH}" + +################################################# +exit 0 +################################################# + bbs_build build -j${PARALLELISM} bbs_build --install=/opt/bb --prefix=/ install popd From 01f79997eaa8250fc975107e94547bda32adaed2 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Wed, 10 Jul 2024 11:38:02 +0300 Subject: [PATCH 042/105] build.yaml: debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/build_sanitizer.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index a672e4733a..689b6fb252 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -140,12 +140,12 @@ export DIR_SCRIPTS="${DIR_SCRIPTS}" ################################################# echo ################################################# -sudo update-alternatives --all +# sudo update-alternatives --all echo ################################################# -sudo update-alternatives --remove-all gcc -sudo update-alternatives --remove-all llvm -sudo update-alternatives --remove-all clang +# sudo update-alternatives --remove-all gcc +# sudo update-alternatives --remove-all llvm +# sudo update-alternatives --remove-all clang # sudo ln -sf /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang # sudo ln -sf /usr/bin/clang++-${LLVM_VERSION} /usr/bin/clang++ @@ -155,7 +155,8 @@ sudo update-alternatives \ --slave /usr/bin/lld lld /usr/bin/lld-18 \ echo ################################################# -sudo update-alternatives --all +# sudo update-alternatives --all +sudo update-alternatives --config clang echo ################################################# ################################################# From 79e095ac040a58d03703fb3971ab5b6148b7ff7d Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Wed, 10 Jul 2024 11:53:57 +0300 Subject: [PATCH 043/105] build.yaml: debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .../workflows/sanitizers/build_sanitizer.sh | 97 ++++++++++++++++++- 1 file changed, 92 insertions(+), 5 deletions(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 689b6fb252..d3a022fd54 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -145,14 +145,101 @@ echo ################################################# # sudo update-alternatives --remove-all gcc # sudo update-alternatives --remove-all llvm -# sudo update-alternatives --remove-all clang +sudo update-alternatives --remove-all clang # sudo ln -sf /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang # sudo ln -sf /usr/bin/clang++-${LLVM_VERSION} /usr/bin/clang++ -sudo update-alternatives \ - --install /usr/bin/clang clang /usr/bin/clang-18 100 \ - --slave /usr/bin/clang++ clang++ /usr/bin/clang++-18 \ - --slave /usr/bin/lld lld /usr/bin/lld-18 \ +# sudo update-alternatives \ +# --install /usr/bin/clang clang /usr/bin/clang-18 100 \ +# --slave /usr/bin/clang++ clang++ /usr/bin/clang++-18 \ +# --slave /usr/bin/lld lld /usr/bin/lld-18 \ + + +function register_clang_version { + local version=$1 + local priority=$2 + + update-alternatives \ + --verbose \ + --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-${version} ${priority} \ + --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-${version} \ + --slave /usr/bin/llvm-as llvm-as /usr/bin/llvm-as-${version} \ + --slave /usr/bin/llvm-bcanalyzer llvm-bcanalyzer /usr/bin/llvm-bcanalyzer-${version} \ + --slave /usr/bin/llvm-c-test llvm-c-test /usr/bin/llvm-c-test-${version} \ + --slave /usr/bin/llvm-cat llvm-cat /usr/bin/llvm-cat-${version} \ + --slave /usr/bin/llvm-cfi-verify llvm-cfi-verify /usr/bin/llvm-cfi-verify-${version} \ + --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-${version} \ + --slave /usr/bin/llvm-cvtres llvm-cvtres /usr/bin/llvm-cvtres-${version} \ + --slave /usr/bin/llvm-cxxdump llvm-cxxdump /usr/bin/llvm-cxxdump-${version} \ + --slave /usr/bin/llvm-cxxfilt llvm-cxxfilt /usr/bin/llvm-cxxfilt-${version} \ + --slave /usr/bin/llvm-diff llvm-diff /usr/bin/llvm-diff-${version} \ + --slave /usr/bin/llvm-dis llvm-dis /usr/bin/llvm-dis-${version} \ + --slave /usr/bin/llvm-dlltool llvm-dlltool /usr/bin/llvm-dlltool-${version} \ + --slave /usr/bin/llvm-dwarfdump llvm-dwarfdump /usr/bin/llvm-dwarfdump-${version} \ + --slave /usr/bin/llvm-dwp llvm-dwp /usr/bin/llvm-dwp-${version} \ + --slave /usr/bin/llvm-exegesis llvm-exegesis /usr/bin/llvm-exegesis-${version} \ + --slave /usr/bin/llvm-extract llvm-extract /usr/bin/llvm-extract-${version} \ + --slave /usr/bin/llvm-lib llvm-lib /usr/bin/llvm-lib-${version} \ + --slave /usr/bin/llvm-link llvm-link /usr/bin/llvm-link-${version} \ + --slave /usr/bin/llvm-lto llvm-lto /usr/bin/llvm-lto-${version} \ + --slave /usr/bin/llvm-lto2 llvm-lto2 /usr/bin/llvm-lto2-${version} \ + --slave /usr/bin/llvm-mc llvm-mc /usr/bin/llvm-mc-${version} \ + --slave /usr/bin/llvm-mca llvm-mca /usr/bin/llvm-mca-${version} \ + --slave /usr/bin/llvm-modextract llvm-modextract /usr/bin/llvm-modextract-${version} \ + --slave /usr/bin/llvm-mt llvm-mt /usr/bin/llvm-mt-${version} \ + --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-${version} \ + --slave /usr/bin/llvm-objcopy llvm-objcopy /usr/bin/llvm-objcopy-${version} \ + --slave /usr/bin/llvm-objdump llvm-objdump /usr/bin/llvm-objdump-${version} \ + --slave /usr/bin/llvm-opt-report llvm-opt-report /usr/bin/llvm-opt-report-${version} \ + --slave /usr/bin/llvm-pdbutil llvm-pdbutil /usr/bin/llvm-pdbutil-${version} \ + --slave /usr/bin/llvm-PerfectShuffle llvm-PerfectShuffle /usr/bin/llvm-PerfectShuffle-${version} \ + --slave /usr/bin/llvm-profdata llvm-profdata /usr/bin/llvm-profdata-${version} \ + --slave /usr/bin/llvm-ranlib llvm-ranlib /usr/bin/llvm-ranlib-${version} \ + --slave /usr/bin/llvm-rc llvm-rc /usr/bin/llvm-rc-${version} \ + --slave /usr/bin/llvm-readelf llvm-readelf /usr/bin/llvm-readelf-${version} \ + --slave /usr/bin/llvm-readobj llvm-readobj /usr/bin/llvm-readobj-${version} \ + --slave /usr/bin/llvm-rtdyld llvm-rtdyld /usr/bin/llvm-rtdyld-${version} \ + --slave /usr/bin/llvm-size llvm-size /usr/bin/llvm-size-${version} \ + --slave /usr/bin/llvm-split llvm-split /usr/bin/llvm-split-${version} \ + --slave /usr/bin/llvm-stress llvm-stress /usr/bin/llvm-stress-${version} \ + --slave /usr/bin/llvm-strings llvm-strings /usr/bin/llvm-strings-${version} \ + --slave /usr/bin/llvm-strip llvm-strip /usr/bin/llvm-strip-${version} \ + --slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-${version} \ + --slave /usr/bin/llvm-tblgen llvm-tblgen /usr/bin/llvm-tblgen-${version} \ + --slave /usr/bin/llvm-undname llvm-undname /usr/bin/llvm-undname-${version} \ + --slave /usr/bin/llvm-xray llvm-xray /usr/bin/llvm-xray-${version} + + + update-alternatives \ + --verbose \ + --install /usr/bin/clang clang /usr/bin/clang-${version} ${priority} \ + --slave /usr/bin/clang++ clang++ /usr/bin/clang++-${version} \ + --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-${version} \ + --slave /usr/bin/clang-cpp clang-cpp /usr/bin/clang-cpp-${version} \ + --slave /usr/bin/clang-cl clang-cl /usr/bin/clang-cl-${version} \ + --slave /usr/bin/clangd clangd /usr/bin/clangd-${version} \ + --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-${version} \ + --slave /usr/bin/clang-check clang-check /usr/bin/clang-check-${version} \ + --slave /usr/bin/clang-query clang-query /usr/bin/clang-query-${version} \ + --slave /usr/bin/asan_symbolize asan_symbolize /usr/bin/asan_symbolize-${version} \ + --slave /usr/bin/bugpoint bugpoint /usr/bin/bugpoint-${version} \ + --slave /usr/bin/dsymutil dsymutil /usr/bin/dsymutil-${version} \ + --slave /usr/bin/lld lld /usr/bin/lld-${version} \ + --slave /usr/bin/ld.lld ld.lld /usr/bin/ld.lld-${version} \ + --slave /usr/bin/lld-link lld-link /usr/bin/lld-link-${version} \ + --slave /usr/bin/llc llc /usr/bin/llc-${version} \ + --slave /usr/bin/lli lli /usr/bin/lli-${version} \ + --slave /usr/bin/obj2yaml obj2yaml /usr/bin/obj2yaml-${version} \ + --slave /usr/bin/opt opt /usr/bin/opt-${version} \ + --slave /usr/bin/sanstats sanstats /usr/bin/sanstats-${version} \ + --slave /usr/bin/verify-uselistorder verify-uselistorder /usr/bin/verify-uselistorder-${version} \ + --slave /usr/bin/wasm-ld wasm-ld /usr/bin/wasm-ld-${version} \ + --slave /usr/bin/yaml2obj yaml2obj /usr/bin/yaml2obj-${version} + +} + +register_clang_version 18 100 + echo ################################################# # sudo update-alternatives --all From 10a3ec4be4798abdd88e741bf10f785fbb8837b3 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Wed, 10 Jul 2024 11:59:52 +0300 Subject: [PATCH 044/105] build.yaml: debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/build_sanitizer.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index d3a022fd54..851fe87a45 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -159,7 +159,7 @@ function register_clang_version { local version=$1 local priority=$2 - update-alternatives \ + sudo update-alternatives \ --verbose \ --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-${version} ${priority} \ --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-${version} \ @@ -210,7 +210,7 @@ function register_clang_version { --slave /usr/bin/llvm-xray llvm-xray /usr/bin/llvm-xray-${version} - update-alternatives \ + sudo update-alternatives \ --verbose \ --install /usr/bin/clang clang /usr/bin/clang-${version} ${priority} \ --slave /usr/bin/clang++ clang++ /usr/bin/clang++-${version} \ From 45e905624ef5746318e0ae4a73a11469fe5201cc Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Wed, 10 Jul 2024 12:08:23 +0300 Subject: [PATCH 045/105] build.yaml: debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/build_sanitizer.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 851fe87a45..ce0ae7fe5f 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -146,6 +146,7 @@ echo ################################################# # sudo update-alternatives --remove-all gcc # sudo update-alternatives --remove-all llvm sudo update-alternatives --remove-all clang +sudo update-alternatives --remove-all clang++ # sudo ln -sf /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang # sudo ln -sf /usr/bin/clang++-${LLVM_VERSION} /usr/bin/clang++ From a939200f94464ed851899563b61173e43cddd0ed Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Wed, 10 Jul 2024 12:24:03 +0300 Subject: [PATCH 046/105] build.yaml: debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .../workflows/sanitizers/build_sanitizer.sh | 103 +++++++++--------- 1 file changed, 54 insertions(+), 49 deletions(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index ce0ae7fe5f..160bc75a66 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -144,9 +144,14 @@ echo ################################################# echo ################################################# # sudo update-alternatives --remove-all gcc +update-alternatives --display clang +update-alternatives --display clang++ +update-alternatives --display clang-format + # sudo update-alternatives --remove-all llvm sudo update-alternatives --remove-all clang sudo update-alternatives --remove-all clang++ +sudo update-alternatives --remove-all clang-format # sudo ln -sf /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang # sudo ln -sf /usr/bin/clang++-${LLVM_VERSION} /usr/bin/clang++ @@ -160,55 +165,55 @@ function register_clang_version { local version=$1 local priority=$2 - sudo update-alternatives \ - --verbose \ - --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-${version} ${priority} \ - --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-${version} \ - --slave /usr/bin/llvm-as llvm-as /usr/bin/llvm-as-${version} \ - --slave /usr/bin/llvm-bcanalyzer llvm-bcanalyzer /usr/bin/llvm-bcanalyzer-${version} \ - --slave /usr/bin/llvm-c-test llvm-c-test /usr/bin/llvm-c-test-${version} \ - --slave /usr/bin/llvm-cat llvm-cat /usr/bin/llvm-cat-${version} \ - --slave /usr/bin/llvm-cfi-verify llvm-cfi-verify /usr/bin/llvm-cfi-verify-${version} \ - --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-${version} \ - --slave /usr/bin/llvm-cvtres llvm-cvtres /usr/bin/llvm-cvtres-${version} \ - --slave /usr/bin/llvm-cxxdump llvm-cxxdump /usr/bin/llvm-cxxdump-${version} \ - --slave /usr/bin/llvm-cxxfilt llvm-cxxfilt /usr/bin/llvm-cxxfilt-${version} \ - --slave /usr/bin/llvm-diff llvm-diff /usr/bin/llvm-diff-${version} \ - --slave /usr/bin/llvm-dis llvm-dis /usr/bin/llvm-dis-${version} \ - --slave /usr/bin/llvm-dlltool llvm-dlltool /usr/bin/llvm-dlltool-${version} \ - --slave /usr/bin/llvm-dwarfdump llvm-dwarfdump /usr/bin/llvm-dwarfdump-${version} \ - --slave /usr/bin/llvm-dwp llvm-dwp /usr/bin/llvm-dwp-${version} \ - --slave /usr/bin/llvm-exegesis llvm-exegesis /usr/bin/llvm-exegesis-${version} \ - --slave /usr/bin/llvm-extract llvm-extract /usr/bin/llvm-extract-${version} \ - --slave /usr/bin/llvm-lib llvm-lib /usr/bin/llvm-lib-${version} \ - --slave /usr/bin/llvm-link llvm-link /usr/bin/llvm-link-${version} \ - --slave /usr/bin/llvm-lto llvm-lto /usr/bin/llvm-lto-${version} \ - --slave /usr/bin/llvm-lto2 llvm-lto2 /usr/bin/llvm-lto2-${version} \ - --slave /usr/bin/llvm-mc llvm-mc /usr/bin/llvm-mc-${version} \ - --slave /usr/bin/llvm-mca llvm-mca /usr/bin/llvm-mca-${version} \ - --slave /usr/bin/llvm-modextract llvm-modextract /usr/bin/llvm-modextract-${version} \ - --slave /usr/bin/llvm-mt llvm-mt /usr/bin/llvm-mt-${version} \ - --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-${version} \ - --slave /usr/bin/llvm-objcopy llvm-objcopy /usr/bin/llvm-objcopy-${version} \ - --slave /usr/bin/llvm-objdump llvm-objdump /usr/bin/llvm-objdump-${version} \ - --slave /usr/bin/llvm-opt-report llvm-opt-report /usr/bin/llvm-opt-report-${version} \ - --slave /usr/bin/llvm-pdbutil llvm-pdbutil /usr/bin/llvm-pdbutil-${version} \ - --slave /usr/bin/llvm-PerfectShuffle llvm-PerfectShuffle /usr/bin/llvm-PerfectShuffle-${version} \ - --slave /usr/bin/llvm-profdata llvm-profdata /usr/bin/llvm-profdata-${version} \ - --slave /usr/bin/llvm-ranlib llvm-ranlib /usr/bin/llvm-ranlib-${version} \ - --slave /usr/bin/llvm-rc llvm-rc /usr/bin/llvm-rc-${version} \ - --slave /usr/bin/llvm-readelf llvm-readelf /usr/bin/llvm-readelf-${version} \ - --slave /usr/bin/llvm-readobj llvm-readobj /usr/bin/llvm-readobj-${version} \ - --slave /usr/bin/llvm-rtdyld llvm-rtdyld /usr/bin/llvm-rtdyld-${version} \ - --slave /usr/bin/llvm-size llvm-size /usr/bin/llvm-size-${version} \ - --slave /usr/bin/llvm-split llvm-split /usr/bin/llvm-split-${version} \ - --slave /usr/bin/llvm-stress llvm-stress /usr/bin/llvm-stress-${version} \ - --slave /usr/bin/llvm-strings llvm-strings /usr/bin/llvm-strings-${version} \ - --slave /usr/bin/llvm-strip llvm-strip /usr/bin/llvm-strip-${version} \ - --slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-${version} \ - --slave /usr/bin/llvm-tblgen llvm-tblgen /usr/bin/llvm-tblgen-${version} \ - --slave /usr/bin/llvm-undname llvm-undname /usr/bin/llvm-undname-${version} \ - --slave /usr/bin/llvm-xray llvm-xray /usr/bin/llvm-xray-${version} + # sudo update-alternatives \ + # --verbose \ + # --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-${version} ${priority} \ + # --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-${version} \ + # --slave /usr/bin/llvm-as llvm-as /usr/bin/llvm-as-${version} \ + # --slave /usr/bin/llvm-bcanalyzer llvm-bcanalyzer /usr/bin/llvm-bcanalyzer-${version} \ + # --slave /usr/bin/llvm-c-test llvm-c-test /usr/bin/llvm-c-test-${version} \ + # --slave /usr/bin/llvm-cat llvm-cat /usr/bin/llvm-cat-${version} \ + # --slave /usr/bin/llvm-cfi-verify llvm-cfi-verify /usr/bin/llvm-cfi-verify-${version} \ + # --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-${version} \ + # --slave /usr/bin/llvm-cvtres llvm-cvtres /usr/bin/llvm-cvtres-${version} \ + # --slave /usr/bin/llvm-cxxdump llvm-cxxdump /usr/bin/llvm-cxxdump-${version} \ + # --slave /usr/bin/llvm-cxxfilt llvm-cxxfilt /usr/bin/llvm-cxxfilt-${version} \ + # --slave /usr/bin/llvm-diff llvm-diff /usr/bin/llvm-diff-${version} \ + # --slave /usr/bin/llvm-dis llvm-dis /usr/bin/llvm-dis-${version} \ + # --slave /usr/bin/llvm-dlltool llvm-dlltool /usr/bin/llvm-dlltool-${version} \ + # --slave /usr/bin/llvm-dwarfdump llvm-dwarfdump /usr/bin/llvm-dwarfdump-${version} \ + # --slave /usr/bin/llvm-dwp llvm-dwp /usr/bin/llvm-dwp-${version} \ + # --slave /usr/bin/llvm-exegesis llvm-exegesis /usr/bin/llvm-exegesis-${version} \ + # --slave /usr/bin/llvm-extract llvm-extract /usr/bin/llvm-extract-${version} \ + # --slave /usr/bin/llvm-lib llvm-lib /usr/bin/llvm-lib-${version} \ + # --slave /usr/bin/llvm-link llvm-link /usr/bin/llvm-link-${version} \ + # --slave /usr/bin/llvm-lto llvm-lto /usr/bin/llvm-lto-${version} \ + # --slave /usr/bin/llvm-lto2 llvm-lto2 /usr/bin/llvm-lto2-${version} \ + # --slave /usr/bin/llvm-mc llvm-mc /usr/bin/llvm-mc-${version} \ + # --slave /usr/bin/llvm-mca llvm-mca /usr/bin/llvm-mca-${version} \ + # --slave /usr/bin/llvm-modextract llvm-modextract /usr/bin/llvm-modextract-${version} \ + # --slave /usr/bin/llvm-mt llvm-mt /usr/bin/llvm-mt-${version} \ + # --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-${version} \ + # --slave /usr/bin/llvm-objcopy llvm-objcopy /usr/bin/llvm-objcopy-${version} \ + # --slave /usr/bin/llvm-objdump llvm-objdump /usr/bin/llvm-objdump-${version} \ + # --slave /usr/bin/llvm-opt-report llvm-opt-report /usr/bin/llvm-opt-report-${version} \ + # --slave /usr/bin/llvm-pdbutil llvm-pdbutil /usr/bin/llvm-pdbutil-${version} \ + # --slave /usr/bin/llvm-PerfectShuffle llvm-PerfectShuffle /usr/bin/llvm-PerfectShuffle-${version} \ + # --slave /usr/bin/llvm-profdata llvm-profdata /usr/bin/llvm-profdata-${version} \ + # --slave /usr/bin/llvm-ranlib llvm-ranlib /usr/bin/llvm-ranlib-${version} \ + # --slave /usr/bin/llvm-rc llvm-rc /usr/bin/llvm-rc-${version} \ + # --slave /usr/bin/llvm-readelf llvm-readelf /usr/bin/llvm-readelf-${version} \ + # --slave /usr/bin/llvm-readobj llvm-readobj /usr/bin/llvm-readobj-${version} \ + # --slave /usr/bin/llvm-rtdyld llvm-rtdyld /usr/bin/llvm-rtdyld-${version} \ + # --slave /usr/bin/llvm-size llvm-size /usr/bin/llvm-size-${version} \ + # --slave /usr/bin/llvm-split llvm-split /usr/bin/llvm-split-${version} \ + # --slave /usr/bin/llvm-stress llvm-stress /usr/bin/llvm-stress-${version} \ + # --slave /usr/bin/llvm-strings llvm-strings /usr/bin/llvm-strings-${version} \ + # --slave /usr/bin/llvm-strip llvm-strip /usr/bin/llvm-strip-${version} \ + # --slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-${version} \ + # --slave /usr/bin/llvm-tblgen llvm-tblgen /usr/bin/llvm-tblgen-${version} \ + # --slave /usr/bin/llvm-undname llvm-undname /usr/bin/llvm-undname-${version} \ + # --slave /usr/bin/llvm-xray llvm-xray /usr/bin/llvm-xray-${version} sudo update-alternatives \ From d4a42317642f0df03c744b6addc5db34251ebb9b Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Wed, 10 Jul 2024 12:32:15 +0300 Subject: [PATCH 047/105] build.yaml: debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .../workflows/sanitizers/build_sanitizer.sh | 119 +++++++++--------- 1 file changed, 61 insertions(+), 58 deletions(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 160bc75a66..b6b9f77473 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -121,16 +121,16 @@ checkoutGitRepo "$(github_url madler/zlib)" "${ZLIB_TAG}" "zlib" LIBCXX_SRC_PATH="${DIR_SRCS_EXT}/llvm-project/runtimes" LIBCXX_BUILD_PATH="${LIBCXX_SRC_PATH}/cmake.bld" -cmake -B "${LIBCXX_BUILD_PATH}" \ - -S "${LIBCXX_SRC_PATH}" \ - -DCMAKE_BUILD_TYPE="Debug" \ - -DCMAKE_C_COMPILER="clang" \ - -DCMAKE_CXX_COMPILER="clang++" \ - -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ - -DLLVM_USE_SANITIZER="${LLVM_SANITIZER_NAME}" \ - ${LLVM_SPECIFIC_CMAKE_OPTIONS} +# cmake -B "${LIBCXX_BUILD_PATH}" \ +# -S "${LIBCXX_SRC_PATH}" \ +# -DCMAKE_BUILD_TYPE="Debug" \ +# -DCMAKE_C_COMPILER="clang" \ +# -DCMAKE_CXX_COMPILER="clang++" \ +# -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ +# -DLLVM_USE_SANITIZER="${LLVM_SANITIZER_NAME}" \ +# ${LLVM_SPECIFIC_CMAKE_OPTIONS} -cmake --build "${LIBCXX_BUILD_PATH}" -j${PARALLELISM} --target cxx cxxabi unwind generate-cxx-headers +# cmake --build "${LIBCXX_BUILD_PATH}" -j${PARALLELISM} --target cxx cxxabi unwind generate-cxx-headers # Variables read by our custom CMake toolchain used to build everything else. export LIBCXX_BUILD_PATH="$(realpath ${LIBCXX_BUILD_PATH})" @@ -143,6 +143,8 @@ echo ################################################# # sudo update-alternatives --all echo ################################################# +sudo apt-get purge llvm-14 clang-14 + # sudo update-alternatives --remove-all gcc update-alternatives --display clang update-alternatives --display clang++ @@ -152,6 +154,7 @@ update-alternatives --display clang-format sudo update-alternatives --remove-all clang sudo update-alternatives --remove-all clang++ sudo update-alternatives --remove-all clang-format +sudo update-alternatives --remove-all clang-tidy # sudo ln -sf /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang # sudo ln -sf /usr/bin/clang++-${LLVM_VERSION} /usr/bin/clang++ @@ -165,55 +168,55 @@ function register_clang_version { local version=$1 local priority=$2 - # sudo update-alternatives \ - # --verbose \ - # --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-${version} ${priority} \ - # --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-${version} \ - # --slave /usr/bin/llvm-as llvm-as /usr/bin/llvm-as-${version} \ - # --slave /usr/bin/llvm-bcanalyzer llvm-bcanalyzer /usr/bin/llvm-bcanalyzer-${version} \ - # --slave /usr/bin/llvm-c-test llvm-c-test /usr/bin/llvm-c-test-${version} \ - # --slave /usr/bin/llvm-cat llvm-cat /usr/bin/llvm-cat-${version} \ - # --slave /usr/bin/llvm-cfi-verify llvm-cfi-verify /usr/bin/llvm-cfi-verify-${version} \ - # --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-${version} \ - # --slave /usr/bin/llvm-cvtres llvm-cvtres /usr/bin/llvm-cvtres-${version} \ - # --slave /usr/bin/llvm-cxxdump llvm-cxxdump /usr/bin/llvm-cxxdump-${version} \ - # --slave /usr/bin/llvm-cxxfilt llvm-cxxfilt /usr/bin/llvm-cxxfilt-${version} \ - # --slave /usr/bin/llvm-diff llvm-diff /usr/bin/llvm-diff-${version} \ - # --slave /usr/bin/llvm-dis llvm-dis /usr/bin/llvm-dis-${version} \ - # --slave /usr/bin/llvm-dlltool llvm-dlltool /usr/bin/llvm-dlltool-${version} \ - # --slave /usr/bin/llvm-dwarfdump llvm-dwarfdump /usr/bin/llvm-dwarfdump-${version} \ - # --slave /usr/bin/llvm-dwp llvm-dwp /usr/bin/llvm-dwp-${version} \ - # --slave /usr/bin/llvm-exegesis llvm-exegesis /usr/bin/llvm-exegesis-${version} \ - # --slave /usr/bin/llvm-extract llvm-extract /usr/bin/llvm-extract-${version} \ - # --slave /usr/bin/llvm-lib llvm-lib /usr/bin/llvm-lib-${version} \ - # --slave /usr/bin/llvm-link llvm-link /usr/bin/llvm-link-${version} \ - # --slave /usr/bin/llvm-lto llvm-lto /usr/bin/llvm-lto-${version} \ - # --slave /usr/bin/llvm-lto2 llvm-lto2 /usr/bin/llvm-lto2-${version} \ - # --slave /usr/bin/llvm-mc llvm-mc /usr/bin/llvm-mc-${version} \ - # --slave /usr/bin/llvm-mca llvm-mca /usr/bin/llvm-mca-${version} \ - # --slave /usr/bin/llvm-modextract llvm-modextract /usr/bin/llvm-modextract-${version} \ - # --slave /usr/bin/llvm-mt llvm-mt /usr/bin/llvm-mt-${version} \ - # --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-${version} \ - # --slave /usr/bin/llvm-objcopy llvm-objcopy /usr/bin/llvm-objcopy-${version} \ - # --slave /usr/bin/llvm-objdump llvm-objdump /usr/bin/llvm-objdump-${version} \ - # --slave /usr/bin/llvm-opt-report llvm-opt-report /usr/bin/llvm-opt-report-${version} \ - # --slave /usr/bin/llvm-pdbutil llvm-pdbutil /usr/bin/llvm-pdbutil-${version} \ - # --slave /usr/bin/llvm-PerfectShuffle llvm-PerfectShuffle /usr/bin/llvm-PerfectShuffle-${version} \ - # --slave /usr/bin/llvm-profdata llvm-profdata /usr/bin/llvm-profdata-${version} \ - # --slave /usr/bin/llvm-ranlib llvm-ranlib /usr/bin/llvm-ranlib-${version} \ - # --slave /usr/bin/llvm-rc llvm-rc /usr/bin/llvm-rc-${version} \ - # --slave /usr/bin/llvm-readelf llvm-readelf /usr/bin/llvm-readelf-${version} \ - # --slave /usr/bin/llvm-readobj llvm-readobj /usr/bin/llvm-readobj-${version} \ - # --slave /usr/bin/llvm-rtdyld llvm-rtdyld /usr/bin/llvm-rtdyld-${version} \ - # --slave /usr/bin/llvm-size llvm-size /usr/bin/llvm-size-${version} \ - # --slave /usr/bin/llvm-split llvm-split /usr/bin/llvm-split-${version} \ - # --slave /usr/bin/llvm-stress llvm-stress /usr/bin/llvm-stress-${version} \ - # --slave /usr/bin/llvm-strings llvm-strings /usr/bin/llvm-strings-${version} \ - # --slave /usr/bin/llvm-strip llvm-strip /usr/bin/llvm-strip-${version} \ - # --slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-${version} \ - # --slave /usr/bin/llvm-tblgen llvm-tblgen /usr/bin/llvm-tblgen-${version} \ - # --slave /usr/bin/llvm-undname llvm-undname /usr/bin/llvm-undname-${version} \ - # --slave /usr/bin/llvm-xray llvm-xray /usr/bin/llvm-xray-${version} + sudo update-alternatives \ + --verbose \ + --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-${version} ${priority} \ + --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-${version} \ + --slave /usr/bin/llvm-as llvm-as /usr/bin/llvm-as-${version} \ + --slave /usr/bin/llvm-bcanalyzer llvm-bcanalyzer /usr/bin/llvm-bcanalyzer-${version} \ + --slave /usr/bin/llvm-c-test llvm-c-test /usr/bin/llvm-c-test-${version} \ + --slave /usr/bin/llvm-cat llvm-cat /usr/bin/llvm-cat-${version} \ + --slave /usr/bin/llvm-cfi-verify llvm-cfi-verify /usr/bin/llvm-cfi-verify-${version} \ + --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-${version} \ + --slave /usr/bin/llvm-cvtres llvm-cvtres /usr/bin/llvm-cvtres-${version} \ + --slave /usr/bin/llvm-cxxdump llvm-cxxdump /usr/bin/llvm-cxxdump-${version} \ + --slave /usr/bin/llvm-cxxfilt llvm-cxxfilt /usr/bin/llvm-cxxfilt-${version} \ + --slave /usr/bin/llvm-diff llvm-diff /usr/bin/llvm-diff-${version} \ + --slave /usr/bin/llvm-dis llvm-dis /usr/bin/llvm-dis-${version} \ + --slave /usr/bin/llvm-dlltool llvm-dlltool /usr/bin/llvm-dlltool-${version} \ + --slave /usr/bin/llvm-dwarfdump llvm-dwarfdump /usr/bin/llvm-dwarfdump-${version} \ + --slave /usr/bin/llvm-dwp llvm-dwp /usr/bin/llvm-dwp-${version} \ + --slave /usr/bin/llvm-exegesis llvm-exegesis /usr/bin/llvm-exegesis-${version} \ + --slave /usr/bin/llvm-extract llvm-extract /usr/bin/llvm-extract-${version} \ + --slave /usr/bin/llvm-lib llvm-lib /usr/bin/llvm-lib-${version} \ + --slave /usr/bin/llvm-link llvm-link /usr/bin/llvm-link-${version} \ + --slave /usr/bin/llvm-lto llvm-lto /usr/bin/llvm-lto-${version} \ + --slave /usr/bin/llvm-lto2 llvm-lto2 /usr/bin/llvm-lto2-${version} \ + --slave /usr/bin/llvm-mc llvm-mc /usr/bin/llvm-mc-${version} \ + --slave /usr/bin/llvm-mca llvm-mca /usr/bin/llvm-mca-${version} \ + --slave /usr/bin/llvm-modextract llvm-modextract /usr/bin/llvm-modextract-${version} \ + --slave /usr/bin/llvm-mt llvm-mt /usr/bin/llvm-mt-${version} \ + --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-${version} \ + --slave /usr/bin/llvm-objcopy llvm-objcopy /usr/bin/llvm-objcopy-${version} \ + --slave /usr/bin/llvm-objdump llvm-objdump /usr/bin/llvm-objdump-${version} \ + --slave /usr/bin/llvm-opt-report llvm-opt-report /usr/bin/llvm-opt-report-${version} \ + --slave /usr/bin/llvm-pdbutil llvm-pdbutil /usr/bin/llvm-pdbutil-${version} \ + --slave /usr/bin/llvm-PerfectShuffle llvm-PerfectShuffle /usr/bin/llvm-PerfectShuffle-${version} \ + --slave /usr/bin/llvm-profdata llvm-profdata /usr/bin/llvm-profdata-${version} \ + --slave /usr/bin/llvm-ranlib llvm-ranlib /usr/bin/llvm-ranlib-${version} \ + --slave /usr/bin/llvm-rc llvm-rc /usr/bin/llvm-rc-${version} \ + --slave /usr/bin/llvm-readelf llvm-readelf /usr/bin/llvm-readelf-${version} \ + --slave /usr/bin/llvm-readobj llvm-readobj /usr/bin/llvm-readobj-${version} \ + --slave /usr/bin/llvm-rtdyld llvm-rtdyld /usr/bin/llvm-rtdyld-${version} \ + --slave /usr/bin/llvm-size llvm-size /usr/bin/llvm-size-${version} \ + --slave /usr/bin/llvm-split llvm-split /usr/bin/llvm-split-${version} \ + --slave /usr/bin/llvm-stress llvm-stress /usr/bin/llvm-stress-${version} \ + --slave /usr/bin/llvm-strings llvm-strings /usr/bin/llvm-strings-${version} \ + --slave /usr/bin/llvm-strip llvm-strip /usr/bin/llvm-strip-${version} \ + --slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-${version} \ + --slave /usr/bin/llvm-tblgen llvm-tblgen /usr/bin/llvm-tblgen-${version} \ + --slave /usr/bin/llvm-undname llvm-undname /usr/bin/llvm-undname-${version} \ + --slave /usr/bin/llvm-xray llvm-xray /usr/bin/llvm-xray-${version} sudo update-alternatives \ From d83b46411d183b511e3bbfd090ded877a4f4e558 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Wed, 10 Jul 2024 12:40:11 +0300 Subject: [PATCH 048/105] build.yaml: debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .../workflows/sanitizers/build_sanitizer.sh | 21 +------------------ 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index b6b9f77473..02450b93cf 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -143,26 +143,7 @@ echo ################################################# # sudo update-alternatives --all echo ################################################# -sudo apt-get purge llvm-14 clang-14 - -# sudo update-alternatives --remove-all gcc -update-alternatives --display clang -update-alternatives --display clang++ -update-alternatives --display clang-format - -# sudo update-alternatives --remove-all llvm -sudo update-alternatives --remove-all clang -sudo update-alternatives --remove-all clang++ -sudo update-alternatives --remove-all clang-format -sudo update-alternatives --remove-all clang-tidy - -# sudo ln -sf /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang -# sudo ln -sf /usr/bin/clang++-${LLVM_VERSION} /usr/bin/clang++ -# sudo update-alternatives \ -# --install /usr/bin/clang clang /usr/bin/clang-18 100 \ -# --slave /usr/bin/clang++ clang++ /usr/bin/clang++-18 \ -# --slave /usr/bin/lld lld /usr/bin/lld-18 \ - +sudo apt-get purge llvm-14 clang-14 gcc-11.4.0 function register_clang_version { local version=$1 From 4732a1bc50cddc39ec20dc2e85dfc9280583d646 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Wed, 10 Jul 2024 12:43:53 +0300 Subject: [PATCH 049/105] build.yaml: debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/build_sanitizer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 02450b93cf..93c991af45 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -143,7 +143,7 @@ echo ################################################# # sudo update-alternatives --all echo ################################################# -sudo apt-get purge llvm-14 clang-14 gcc-11.4.0 +sudo apt-get purge llvm-14 clang-14 gcc-11.4 function register_clang_version { local version=$1 From 2801c05f3f7adf04cf80b2d28f2ecb3367d10752 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Wed, 10 Jul 2024 12:47:34 +0300 Subject: [PATCH 050/105] build.yaml: debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/build_sanitizer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 93c991af45..53040b8593 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -143,7 +143,7 @@ echo ################################################# # sudo update-alternatives --all echo ################################################# -sudo apt-get purge llvm-14 clang-14 gcc-11.4 +sudo apt-get purge llvm-14 clang-14 gcc-11 function register_clang_version { local version=$1 From ed275eb5ec08a3ec1afc08fd4e0ad8f3a2d072e0 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Wed, 10 Jul 2024 12:52:03 +0300 Subject: [PATCH 051/105] build.yaml: debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .../workflows/sanitizers/build_sanitizer.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 53040b8593..771838564d 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -145,6 +145,25 @@ echo ################################################# sudo apt-get purge llvm-14 clang-14 gcc-11 +# sudo update-alternatives --remove-all gcc +update-alternatives --display clang +update-alternatives --display clang++ +update-alternatives --display clang-format + +# sudo update-alternatives --remove-all llvm +sudo update-alternatives --remove-all clang +sudo update-alternatives --remove-all clang++ +sudo update-alternatives --remove-all clang-format +sudo update-alternatives --remove-all clang-tidy + +# sudo ln -sf /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang +# sudo ln -sf /usr/bin/clang++-${LLVM_VERSION} /usr/bin/clang++ +# sudo update-alternatives \ +# --install /usr/bin/clang clang /usr/bin/clang-18 100 \ +# --slave /usr/bin/clang++ clang++ /usr/bin/clang++-18 \ +# --slave /usr/bin/lld lld /usr/bin/lld-18 \ + + function register_clang_version { local version=$1 local priority=$2 From 528004efb11707d3cb0af829acb5aec3fc5a3d3e Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Wed, 10 Jul 2024 12:56:13 +0300 Subject: [PATCH 052/105] build.yaml: debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/build_sanitizer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 771838564d..3cf207b11b 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -143,7 +143,7 @@ echo ################################################# # sudo update-alternatives --all echo ################################################# -sudo apt-get purge llvm-14 clang-14 gcc-11 +sudo apt-get purge llvm-14 clang-14 gcc-11 gcc-9 # sudo update-alternatives --remove-all gcc update-alternatives --display clang From f6ec3a4fd324b8f906078221d2fa06286f10c983 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Wed, 10 Jul 2024 13:00:20 +0300 Subject: [PATCH 053/105] build.yaml: debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/build_sanitizer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 3cf207b11b..3810ecaa80 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -143,7 +143,7 @@ echo ################################################# # sudo update-alternatives --all echo ################################################# -sudo apt-get purge llvm-14 clang-14 gcc-11 gcc-9 +sudo apt-get purge llvm-14 clang-14 gcc-9 gcc-10 gcc-11 gcc-12 gcc-13 # sudo update-alternatives --remove-all gcc update-alternatives --display clang From 7f04b571820f43d9f543779df4dcb753cb77c377 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Wed, 10 Jul 2024 13:47:25 +0300 Subject: [PATCH 054/105] build.yaml: debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/build_sanitizer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 3810ecaa80..ac4224a257 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -143,7 +143,7 @@ echo ################################################# # sudo update-alternatives --all echo ################################################# -sudo apt-get purge llvm-14 clang-14 gcc-9 gcc-10 gcc-11 gcc-12 gcc-13 +sudo apt-get purge llvm-14 clang-14 gcc-9 gcc-10 gcc-11 gcc-12 # sudo update-alternatives --remove-all gcc update-alternatives --display clang From ed85cdde2722258bfb1b557b2e77c27cfb5c573e Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Wed, 10 Jul 2024 14:50:02 +0300 Subject: [PATCH 055/105] build.yaml: debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/build_sanitizer.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index ac4224a257..84949d7449 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -269,15 +269,12 @@ pushd ${DIR_SRCS_EXT} pushd "bde" eval "$(bbs_build_env -u dbg_64_safe_cpp20 -b "${DIR_BUILD_EXT}/bde")" bbs_build configure --toolchain "${TOOLCHAIN_PATH}" - -################################################# -exit 0 -################################################# - bbs_build build -j${PARALLELISM} bbs_build --install=/opt/bb --prefix=/ install popd +exit 0 + pushd "ntf-core" # TODO The deprecated flag "-fcoroutines-ts" has been removed in clang # 17.0.1, but NTF is still using it. We manually change this flag until From 5e74621fe4ea2057f13fb74ef966e12ca3aa7f33 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Wed, 10 Jul 2024 15:00:00 +0300 Subject: [PATCH 056/105] build.yaml: debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .../workflows/sanitizers/build_sanitizer.sh | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 84949d7449..472c2a1205 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -121,16 +121,16 @@ checkoutGitRepo "$(github_url madler/zlib)" "${ZLIB_TAG}" "zlib" LIBCXX_SRC_PATH="${DIR_SRCS_EXT}/llvm-project/runtimes" LIBCXX_BUILD_PATH="${LIBCXX_SRC_PATH}/cmake.bld" -# cmake -B "${LIBCXX_BUILD_PATH}" \ -# -S "${LIBCXX_SRC_PATH}" \ -# -DCMAKE_BUILD_TYPE="Debug" \ -# -DCMAKE_C_COMPILER="clang" \ -# -DCMAKE_CXX_COMPILER="clang++" \ -# -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ -# -DLLVM_USE_SANITIZER="${LLVM_SANITIZER_NAME}" \ -# ${LLVM_SPECIFIC_CMAKE_OPTIONS} - -# cmake --build "${LIBCXX_BUILD_PATH}" -j${PARALLELISM} --target cxx cxxabi unwind generate-cxx-headers +cmake -B "${LIBCXX_BUILD_PATH}" \ + -S "${LIBCXX_SRC_PATH}" \ + -DCMAKE_BUILD_TYPE="Debug" \ + -DCMAKE_C_COMPILER="clang" \ + -DCMAKE_CXX_COMPILER="clang++" \ + -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ + -DLLVM_USE_SANITIZER="${LLVM_SANITIZER_NAME}" \ + ${LLVM_SPECIFIC_CMAKE_OPTIONS} + +cmake --build "${LIBCXX_BUILD_PATH}" -j${PARALLELISM} --target cxx cxxabi unwind generate-cxx-headers # Variables read by our custom CMake toolchain used to build everything else. export LIBCXX_BUILD_PATH="$(realpath ${LIBCXX_BUILD_PATH})" From fa5d6579c12d3faade4a0e17c0c9c62617b5316b Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Wed, 10 Jul 2024 15:20:45 +0300 Subject: [PATCH 057/105] build.yaml: debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/build_sanitizer.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 472c2a1205..32265e01d9 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -273,8 +273,6 @@ bbs_build build -j${PARALLELISM} bbs_build --install=/opt/bb --prefix=/ install popd -exit 0 - pushd "ntf-core" # TODO The deprecated flag "-fcoroutines-ts" has been removed in clang # 17.0.1, but NTF is still using it. We manually change this flag until From db38a033fa5834ab6c7264b224ba67ede58067e0 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Wed, 10 Jul 2024 15:59:01 +0300 Subject: [PATCH 058/105] build.yaml: debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .../workflows/sanitizers/build_sanitizer.sh | 208 +++++++++--------- 1 file changed, 104 insertions(+), 104 deletions(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 32265e01d9..af35738130 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -145,116 +145,116 @@ echo ################################################# sudo apt-get purge llvm-14 clang-14 gcc-9 gcc-10 gcc-11 gcc-12 -# sudo update-alternatives --remove-all gcc -update-alternatives --display clang -update-alternatives --display clang++ -update-alternatives --display clang-format - -# sudo update-alternatives --remove-all llvm -sudo update-alternatives --remove-all clang -sudo update-alternatives --remove-all clang++ -sudo update-alternatives --remove-all clang-format -sudo update-alternatives --remove-all clang-tidy - -# sudo ln -sf /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang -# sudo ln -sf /usr/bin/clang++-${LLVM_VERSION} /usr/bin/clang++ -# sudo update-alternatives \ -# --install /usr/bin/clang clang /usr/bin/clang-18 100 \ -# --slave /usr/bin/clang++ clang++ /usr/bin/clang++-18 \ -# --slave /usr/bin/lld lld /usr/bin/lld-18 \ - - -function register_clang_version { - local version=$1 - local priority=$2 - - sudo update-alternatives \ - --verbose \ - --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-${version} ${priority} \ - --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-${version} \ - --slave /usr/bin/llvm-as llvm-as /usr/bin/llvm-as-${version} \ - --slave /usr/bin/llvm-bcanalyzer llvm-bcanalyzer /usr/bin/llvm-bcanalyzer-${version} \ - --slave /usr/bin/llvm-c-test llvm-c-test /usr/bin/llvm-c-test-${version} \ - --slave /usr/bin/llvm-cat llvm-cat /usr/bin/llvm-cat-${version} \ - --slave /usr/bin/llvm-cfi-verify llvm-cfi-verify /usr/bin/llvm-cfi-verify-${version} \ - --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-${version} \ - --slave /usr/bin/llvm-cvtres llvm-cvtres /usr/bin/llvm-cvtres-${version} \ - --slave /usr/bin/llvm-cxxdump llvm-cxxdump /usr/bin/llvm-cxxdump-${version} \ - --slave /usr/bin/llvm-cxxfilt llvm-cxxfilt /usr/bin/llvm-cxxfilt-${version} \ - --slave /usr/bin/llvm-diff llvm-diff /usr/bin/llvm-diff-${version} \ - --slave /usr/bin/llvm-dis llvm-dis /usr/bin/llvm-dis-${version} \ - --slave /usr/bin/llvm-dlltool llvm-dlltool /usr/bin/llvm-dlltool-${version} \ - --slave /usr/bin/llvm-dwarfdump llvm-dwarfdump /usr/bin/llvm-dwarfdump-${version} \ - --slave /usr/bin/llvm-dwp llvm-dwp /usr/bin/llvm-dwp-${version} \ - --slave /usr/bin/llvm-exegesis llvm-exegesis /usr/bin/llvm-exegesis-${version} \ - --slave /usr/bin/llvm-extract llvm-extract /usr/bin/llvm-extract-${version} \ - --slave /usr/bin/llvm-lib llvm-lib /usr/bin/llvm-lib-${version} \ - --slave /usr/bin/llvm-link llvm-link /usr/bin/llvm-link-${version} \ - --slave /usr/bin/llvm-lto llvm-lto /usr/bin/llvm-lto-${version} \ - --slave /usr/bin/llvm-lto2 llvm-lto2 /usr/bin/llvm-lto2-${version} \ - --slave /usr/bin/llvm-mc llvm-mc /usr/bin/llvm-mc-${version} \ - --slave /usr/bin/llvm-mca llvm-mca /usr/bin/llvm-mca-${version} \ - --slave /usr/bin/llvm-modextract llvm-modextract /usr/bin/llvm-modextract-${version} \ - --slave /usr/bin/llvm-mt llvm-mt /usr/bin/llvm-mt-${version} \ - --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-${version} \ - --slave /usr/bin/llvm-objcopy llvm-objcopy /usr/bin/llvm-objcopy-${version} \ - --slave /usr/bin/llvm-objdump llvm-objdump /usr/bin/llvm-objdump-${version} \ - --slave /usr/bin/llvm-opt-report llvm-opt-report /usr/bin/llvm-opt-report-${version} \ - --slave /usr/bin/llvm-pdbutil llvm-pdbutil /usr/bin/llvm-pdbutil-${version} \ - --slave /usr/bin/llvm-PerfectShuffle llvm-PerfectShuffle /usr/bin/llvm-PerfectShuffle-${version} \ - --slave /usr/bin/llvm-profdata llvm-profdata /usr/bin/llvm-profdata-${version} \ - --slave /usr/bin/llvm-ranlib llvm-ranlib /usr/bin/llvm-ranlib-${version} \ - --slave /usr/bin/llvm-rc llvm-rc /usr/bin/llvm-rc-${version} \ - --slave /usr/bin/llvm-readelf llvm-readelf /usr/bin/llvm-readelf-${version} \ - --slave /usr/bin/llvm-readobj llvm-readobj /usr/bin/llvm-readobj-${version} \ - --slave /usr/bin/llvm-rtdyld llvm-rtdyld /usr/bin/llvm-rtdyld-${version} \ - --slave /usr/bin/llvm-size llvm-size /usr/bin/llvm-size-${version} \ - --slave /usr/bin/llvm-split llvm-split /usr/bin/llvm-split-${version} \ - --slave /usr/bin/llvm-stress llvm-stress /usr/bin/llvm-stress-${version} \ - --slave /usr/bin/llvm-strings llvm-strings /usr/bin/llvm-strings-${version} \ - --slave /usr/bin/llvm-strip llvm-strip /usr/bin/llvm-strip-${version} \ - --slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-${version} \ - --slave /usr/bin/llvm-tblgen llvm-tblgen /usr/bin/llvm-tblgen-${version} \ - --slave /usr/bin/llvm-undname llvm-undname /usr/bin/llvm-undname-${version} \ - --slave /usr/bin/llvm-xray llvm-xray /usr/bin/llvm-xray-${version} +# # sudo update-alternatives --remove-all gcc +# update-alternatives --display clang +# update-alternatives --display clang++ +# update-alternatives --display clang-format + +# # sudo update-alternatives --remove-all llvm +# sudo update-alternatives --remove-all clang +# sudo update-alternatives --remove-all clang++ +# sudo update-alternatives --remove-all clang-format +# sudo update-alternatives --remove-all clang-tidy + +# # sudo ln -sf /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang +# # sudo ln -sf /usr/bin/clang++-${LLVM_VERSION} /usr/bin/clang++ +# # sudo update-alternatives \ +# # --install /usr/bin/clang clang /usr/bin/clang-18 100 \ +# # --slave /usr/bin/clang++ clang++ /usr/bin/clang++-18 \ +# # --slave /usr/bin/lld lld /usr/bin/lld-18 \ + + +# function register_clang_version { +# local version=$1 +# local priority=$2 + +# sudo update-alternatives \ +# --verbose \ +# --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-${version} ${priority} \ +# --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-${version} \ +# --slave /usr/bin/llvm-as llvm-as /usr/bin/llvm-as-${version} \ +# --slave /usr/bin/llvm-bcanalyzer llvm-bcanalyzer /usr/bin/llvm-bcanalyzer-${version} \ +# --slave /usr/bin/llvm-c-test llvm-c-test /usr/bin/llvm-c-test-${version} \ +# --slave /usr/bin/llvm-cat llvm-cat /usr/bin/llvm-cat-${version} \ +# --slave /usr/bin/llvm-cfi-verify llvm-cfi-verify /usr/bin/llvm-cfi-verify-${version} \ +# --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-${version} \ +# --slave /usr/bin/llvm-cvtres llvm-cvtres /usr/bin/llvm-cvtres-${version} \ +# --slave /usr/bin/llvm-cxxdump llvm-cxxdump /usr/bin/llvm-cxxdump-${version} \ +# --slave /usr/bin/llvm-cxxfilt llvm-cxxfilt /usr/bin/llvm-cxxfilt-${version} \ +# --slave /usr/bin/llvm-diff llvm-diff /usr/bin/llvm-diff-${version} \ +# --slave /usr/bin/llvm-dis llvm-dis /usr/bin/llvm-dis-${version} \ +# --slave /usr/bin/llvm-dlltool llvm-dlltool /usr/bin/llvm-dlltool-${version} \ +# --slave /usr/bin/llvm-dwarfdump llvm-dwarfdump /usr/bin/llvm-dwarfdump-${version} \ +# --slave /usr/bin/llvm-dwp llvm-dwp /usr/bin/llvm-dwp-${version} \ +# --slave /usr/bin/llvm-exegesis llvm-exegesis /usr/bin/llvm-exegesis-${version} \ +# --slave /usr/bin/llvm-extract llvm-extract /usr/bin/llvm-extract-${version} \ +# --slave /usr/bin/llvm-lib llvm-lib /usr/bin/llvm-lib-${version} \ +# --slave /usr/bin/llvm-link llvm-link /usr/bin/llvm-link-${version} \ +# --slave /usr/bin/llvm-lto llvm-lto /usr/bin/llvm-lto-${version} \ +# --slave /usr/bin/llvm-lto2 llvm-lto2 /usr/bin/llvm-lto2-${version} \ +# --slave /usr/bin/llvm-mc llvm-mc /usr/bin/llvm-mc-${version} \ +# --slave /usr/bin/llvm-mca llvm-mca /usr/bin/llvm-mca-${version} \ +# --slave /usr/bin/llvm-modextract llvm-modextract /usr/bin/llvm-modextract-${version} \ +# --slave /usr/bin/llvm-mt llvm-mt /usr/bin/llvm-mt-${version} \ +# --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-${version} \ +# --slave /usr/bin/llvm-objcopy llvm-objcopy /usr/bin/llvm-objcopy-${version} \ +# --slave /usr/bin/llvm-objdump llvm-objdump /usr/bin/llvm-objdump-${version} \ +# --slave /usr/bin/llvm-opt-report llvm-opt-report /usr/bin/llvm-opt-report-${version} \ +# --slave /usr/bin/llvm-pdbutil llvm-pdbutil /usr/bin/llvm-pdbutil-${version} \ +# --slave /usr/bin/llvm-PerfectShuffle llvm-PerfectShuffle /usr/bin/llvm-PerfectShuffle-${version} \ +# --slave /usr/bin/llvm-profdata llvm-profdata /usr/bin/llvm-profdata-${version} \ +# --slave /usr/bin/llvm-ranlib llvm-ranlib /usr/bin/llvm-ranlib-${version} \ +# --slave /usr/bin/llvm-rc llvm-rc /usr/bin/llvm-rc-${version} \ +# --slave /usr/bin/llvm-readelf llvm-readelf /usr/bin/llvm-readelf-${version} \ +# --slave /usr/bin/llvm-readobj llvm-readobj /usr/bin/llvm-readobj-${version} \ +# --slave /usr/bin/llvm-rtdyld llvm-rtdyld /usr/bin/llvm-rtdyld-${version} \ +# --slave /usr/bin/llvm-size llvm-size /usr/bin/llvm-size-${version} \ +# --slave /usr/bin/llvm-split llvm-split /usr/bin/llvm-split-${version} \ +# --slave /usr/bin/llvm-stress llvm-stress /usr/bin/llvm-stress-${version} \ +# --slave /usr/bin/llvm-strings llvm-strings /usr/bin/llvm-strings-${version} \ +# --slave /usr/bin/llvm-strip llvm-strip /usr/bin/llvm-strip-${version} \ +# --slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-${version} \ +# --slave /usr/bin/llvm-tblgen llvm-tblgen /usr/bin/llvm-tblgen-${version} \ +# --slave /usr/bin/llvm-undname llvm-undname /usr/bin/llvm-undname-${version} \ +# --slave /usr/bin/llvm-xray llvm-xray /usr/bin/llvm-xray-${version} - sudo update-alternatives \ - --verbose \ - --install /usr/bin/clang clang /usr/bin/clang-${version} ${priority} \ - --slave /usr/bin/clang++ clang++ /usr/bin/clang++-${version} \ - --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-${version} \ - --slave /usr/bin/clang-cpp clang-cpp /usr/bin/clang-cpp-${version} \ - --slave /usr/bin/clang-cl clang-cl /usr/bin/clang-cl-${version} \ - --slave /usr/bin/clangd clangd /usr/bin/clangd-${version} \ - --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-${version} \ - --slave /usr/bin/clang-check clang-check /usr/bin/clang-check-${version} \ - --slave /usr/bin/clang-query clang-query /usr/bin/clang-query-${version} \ - --slave /usr/bin/asan_symbolize asan_symbolize /usr/bin/asan_symbolize-${version} \ - --slave /usr/bin/bugpoint bugpoint /usr/bin/bugpoint-${version} \ - --slave /usr/bin/dsymutil dsymutil /usr/bin/dsymutil-${version} \ - --slave /usr/bin/lld lld /usr/bin/lld-${version} \ - --slave /usr/bin/ld.lld ld.lld /usr/bin/ld.lld-${version} \ - --slave /usr/bin/lld-link lld-link /usr/bin/lld-link-${version} \ - --slave /usr/bin/llc llc /usr/bin/llc-${version} \ - --slave /usr/bin/lli lli /usr/bin/lli-${version} \ - --slave /usr/bin/obj2yaml obj2yaml /usr/bin/obj2yaml-${version} \ - --slave /usr/bin/opt opt /usr/bin/opt-${version} \ - --slave /usr/bin/sanstats sanstats /usr/bin/sanstats-${version} \ - --slave /usr/bin/verify-uselistorder verify-uselistorder /usr/bin/verify-uselistorder-${version} \ - --slave /usr/bin/wasm-ld wasm-ld /usr/bin/wasm-ld-${version} \ - --slave /usr/bin/yaml2obj yaml2obj /usr/bin/yaml2obj-${version} +# sudo update-alternatives \ +# --verbose \ +# --install /usr/bin/clang clang /usr/bin/clang-${version} ${priority} \ +# --slave /usr/bin/clang++ clang++ /usr/bin/clang++-${version} \ +# --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-${version} \ +# --slave /usr/bin/clang-cpp clang-cpp /usr/bin/clang-cpp-${version} \ +# --slave /usr/bin/clang-cl clang-cl /usr/bin/clang-cl-${version} \ +# --slave /usr/bin/clangd clangd /usr/bin/clangd-${version} \ +# --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-${version} \ +# --slave /usr/bin/clang-check clang-check /usr/bin/clang-check-${version} \ +# --slave /usr/bin/clang-query clang-query /usr/bin/clang-query-${version} \ +# --slave /usr/bin/asan_symbolize asan_symbolize /usr/bin/asan_symbolize-${version} \ +# --slave /usr/bin/bugpoint bugpoint /usr/bin/bugpoint-${version} \ +# --slave /usr/bin/dsymutil dsymutil /usr/bin/dsymutil-${version} \ +# --slave /usr/bin/lld lld /usr/bin/lld-${version} \ +# --slave /usr/bin/ld.lld ld.lld /usr/bin/ld.lld-${version} \ +# --slave /usr/bin/lld-link lld-link /usr/bin/lld-link-${version} \ +# --slave /usr/bin/llc llc /usr/bin/llc-${version} \ +# --slave /usr/bin/lli lli /usr/bin/lli-${version} \ +# --slave /usr/bin/obj2yaml obj2yaml /usr/bin/obj2yaml-${version} \ +# --slave /usr/bin/opt opt /usr/bin/opt-${version} \ +# --slave /usr/bin/sanstats sanstats /usr/bin/sanstats-${version} \ +# --slave /usr/bin/verify-uselistorder verify-uselistorder /usr/bin/verify-uselistorder-${version} \ +# --slave /usr/bin/wasm-ld wasm-ld /usr/bin/wasm-ld-${version} \ +# --slave /usr/bin/yaml2obj yaml2obj /usr/bin/yaml2obj-${version} -} +# } -register_clang_version 18 100 +# register_clang_version 18 100 -echo ################################################# -# sudo update-alternatives --all -sudo update-alternatives --config clang -echo ################################################# -################################################# +# echo ################################################# +# # sudo update-alternatives --all +# sudo update-alternatives --config clang +# echo ################################################# +# ################################################# # :: Build BDE + NTF :::::::::::::::::::::::::::::::::::::::::::::::::::::::::: TOOLCHAIN_PATH="${DIR_SCRIPTS}/clang-libcxx-${SANITIZER_NAME}.cmake" From 88a2e9ce6ff61365aa15b57829e4263a5967a612 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Wed, 10 Jul 2024 16:21:28 +0300 Subject: [PATCH 059/105] build.yaml: debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .../workflows/sanitizers/build_sanitizer.sh | 168 +++++------------- 1 file changed, 47 insertions(+), 121 deletions(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index af35738130..681808eb64 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -31,8 +31,8 @@ fi SANITIZER_NAME="${1}" -echo SANITIZER_NAME: "${SANITIZER_NAME}" -echo ROOT: "${PWD}" +# Uninstall uneeded tools which cause of versions clash +sudo apt-get purge llvm-14 clang-14 gcc-9 gcc-10 gcc-11 gcc-12 # Install prerequisites sudo apt-get update && sudo apt-get install -qy lsb-release wget software-properties-common gnupg git curl jq ninja-build bison libfl-dev pkg-config @@ -137,125 +137,6 @@ export LIBCXX_BUILD_PATH="$(realpath ${LIBCXX_BUILD_PATH})" export DIR_SRC_BMQ="${DIR_SRC_BMQ}" export DIR_SCRIPTS="${DIR_SCRIPTS}" - -################################################# -echo ################################################# -# sudo update-alternatives --all -echo ################################################# - -sudo apt-get purge llvm-14 clang-14 gcc-9 gcc-10 gcc-11 gcc-12 - -# # sudo update-alternatives --remove-all gcc -# update-alternatives --display clang -# update-alternatives --display clang++ -# update-alternatives --display clang-format - -# # sudo update-alternatives --remove-all llvm -# sudo update-alternatives --remove-all clang -# sudo update-alternatives --remove-all clang++ -# sudo update-alternatives --remove-all clang-format -# sudo update-alternatives --remove-all clang-tidy - -# # sudo ln -sf /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang -# # sudo ln -sf /usr/bin/clang++-${LLVM_VERSION} /usr/bin/clang++ -# # sudo update-alternatives \ -# # --install /usr/bin/clang clang /usr/bin/clang-18 100 \ -# # --slave /usr/bin/clang++ clang++ /usr/bin/clang++-18 \ -# # --slave /usr/bin/lld lld /usr/bin/lld-18 \ - - -# function register_clang_version { -# local version=$1 -# local priority=$2 - -# sudo update-alternatives \ -# --verbose \ -# --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-${version} ${priority} \ -# --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-${version} \ -# --slave /usr/bin/llvm-as llvm-as /usr/bin/llvm-as-${version} \ -# --slave /usr/bin/llvm-bcanalyzer llvm-bcanalyzer /usr/bin/llvm-bcanalyzer-${version} \ -# --slave /usr/bin/llvm-c-test llvm-c-test /usr/bin/llvm-c-test-${version} \ -# --slave /usr/bin/llvm-cat llvm-cat /usr/bin/llvm-cat-${version} \ -# --slave /usr/bin/llvm-cfi-verify llvm-cfi-verify /usr/bin/llvm-cfi-verify-${version} \ -# --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-${version} \ -# --slave /usr/bin/llvm-cvtres llvm-cvtres /usr/bin/llvm-cvtres-${version} \ -# --slave /usr/bin/llvm-cxxdump llvm-cxxdump /usr/bin/llvm-cxxdump-${version} \ -# --slave /usr/bin/llvm-cxxfilt llvm-cxxfilt /usr/bin/llvm-cxxfilt-${version} \ -# --slave /usr/bin/llvm-diff llvm-diff /usr/bin/llvm-diff-${version} \ -# --slave /usr/bin/llvm-dis llvm-dis /usr/bin/llvm-dis-${version} \ -# --slave /usr/bin/llvm-dlltool llvm-dlltool /usr/bin/llvm-dlltool-${version} \ -# --slave /usr/bin/llvm-dwarfdump llvm-dwarfdump /usr/bin/llvm-dwarfdump-${version} \ -# --slave /usr/bin/llvm-dwp llvm-dwp /usr/bin/llvm-dwp-${version} \ -# --slave /usr/bin/llvm-exegesis llvm-exegesis /usr/bin/llvm-exegesis-${version} \ -# --slave /usr/bin/llvm-extract llvm-extract /usr/bin/llvm-extract-${version} \ -# --slave /usr/bin/llvm-lib llvm-lib /usr/bin/llvm-lib-${version} \ -# --slave /usr/bin/llvm-link llvm-link /usr/bin/llvm-link-${version} \ -# --slave /usr/bin/llvm-lto llvm-lto /usr/bin/llvm-lto-${version} \ -# --slave /usr/bin/llvm-lto2 llvm-lto2 /usr/bin/llvm-lto2-${version} \ -# --slave /usr/bin/llvm-mc llvm-mc /usr/bin/llvm-mc-${version} \ -# --slave /usr/bin/llvm-mca llvm-mca /usr/bin/llvm-mca-${version} \ -# --slave /usr/bin/llvm-modextract llvm-modextract /usr/bin/llvm-modextract-${version} \ -# --slave /usr/bin/llvm-mt llvm-mt /usr/bin/llvm-mt-${version} \ -# --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-${version} \ -# --slave /usr/bin/llvm-objcopy llvm-objcopy /usr/bin/llvm-objcopy-${version} \ -# --slave /usr/bin/llvm-objdump llvm-objdump /usr/bin/llvm-objdump-${version} \ -# --slave /usr/bin/llvm-opt-report llvm-opt-report /usr/bin/llvm-opt-report-${version} \ -# --slave /usr/bin/llvm-pdbutil llvm-pdbutil /usr/bin/llvm-pdbutil-${version} \ -# --slave /usr/bin/llvm-PerfectShuffle llvm-PerfectShuffle /usr/bin/llvm-PerfectShuffle-${version} \ -# --slave /usr/bin/llvm-profdata llvm-profdata /usr/bin/llvm-profdata-${version} \ -# --slave /usr/bin/llvm-ranlib llvm-ranlib /usr/bin/llvm-ranlib-${version} \ -# --slave /usr/bin/llvm-rc llvm-rc /usr/bin/llvm-rc-${version} \ -# --slave /usr/bin/llvm-readelf llvm-readelf /usr/bin/llvm-readelf-${version} \ -# --slave /usr/bin/llvm-readobj llvm-readobj /usr/bin/llvm-readobj-${version} \ -# --slave /usr/bin/llvm-rtdyld llvm-rtdyld /usr/bin/llvm-rtdyld-${version} \ -# --slave /usr/bin/llvm-size llvm-size /usr/bin/llvm-size-${version} \ -# --slave /usr/bin/llvm-split llvm-split /usr/bin/llvm-split-${version} \ -# --slave /usr/bin/llvm-stress llvm-stress /usr/bin/llvm-stress-${version} \ -# --slave /usr/bin/llvm-strings llvm-strings /usr/bin/llvm-strings-${version} \ -# --slave /usr/bin/llvm-strip llvm-strip /usr/bin/llvm-strip-${version} \ -# --slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-${version} \ -# --slave /usr/bin/llvm-tblgen llvm-tblgen /usr/bin/llvm-tblgen-${version} \ -# --slave /usr/bin/llvm-undname llvm-undname /usr/bin/llvm-undname-${version} \ -# --slave /usr/bin/llvm-xray llvm-xray /usr/bin/llvm-xray-${version} - - -# sudo update-alternatives \ -# --verbose \ -# --install /usr/bin/clang clang /usr/bin/clang-${version} ${priority} \ -# --slave /usr/bin/clang++ clang++ /usr/bin/clang++-${version} \ -# --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-${version} \ -# --slave /usr/bin/clang-cpp clang-cpp /usr/bin/clang-cpp-${version} \ -# --slave /usr/bin/clang-cl clang-cl /usr/bin/clang-cl-${version} \ -# --slave /usr/bin/clangd clangd /usr/bin/clangd-${version} \ -# --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-${version} \ -# --slave /usr/bin/clang-check clang-check /usr/bin/clang-check-${version} \ -# --slave /usr/bin/clang-query clang-query /usr/bin/clang-query-${version} \ -# --slave /usr/bin/asan_symbolize asan_symbolize /usr/bin/asan_symbolize-${version} \ -# --slave /usr/bin/bugpoint bugpoint /usr/bin/bugpoint-${version} \ -# --slave /usr/bin/dsymutil dsymutil /usr/bin/dsymutil-${version} \ -# --slave /usr/bin/lld lld /usr/bin/lld-${version} \ -# --slave /usr/bin/ld.lld ld.lld /usr/bin/ld.lld-${version} \ -# --slave /usr/bin/lld-link lld-link /usr/bin/lld-link-${version} \ -# --slave /usr/bin/llc llc /usr/bin/llc-${version} \ -# --slave /usr/bin/lli lli /usr/bin/lli-${version} \ -# --slave /usr/bin/obj2yaml obj2yaml /usr/bin/obj2yaml-${version} \ -# --slave /usr/bin/opt opt /usr/bin/opt-${version} \ -# --slave /usr/bin/sanstats sanstats /usr/bin/sanstats-${version} \ -# --slave /usr/bin/verify-uselistorder verify-uselistorder /usr/bin/verify-uselistorder-${version} \ -# --slave /usr/bin/wasm-ld wasm-ld /usr/bin/wasm-ld-${version} \ -# --slave /usr/bin/yaml2obj yaml2obj /usr/bin/yaml2obj-${version} - -# } - -# register_clang_version 18 100 - - -# echo ################################################# -# # sudo update-alternatives --all -# sudo update-alternatives --config clang -# echo ################################################# -# ################################################# - # :: Build BDE + NTF :::::::::::::::::::::::::::::::::::::::::::::::::::::::::: TOOLCHAIN_PATH="${DIR_SCRIPTS}/clang-libcxx-${SANITIZER_NAME}.cmake" export CC="clang" @@ -290,3 +171,48 @@ sed -i 's/fcoroutines-ts/fcoroutines/g' 'repository.cmake' make -j${PARALLELISM} make install popd + +# Note: Hack to circumvent faulty behavior in "nts-targets.cmake" +ln -sf "/opt/bb/include" "/opt/include" +ln -sf "/opt/bb/lib64" "/opt/lib64" + +# pushd DIR_SRCS_EXT +popd + +# :: Setup CMake options for all remaining builds ::::::::::::::::::::::::::::: +CMAKE_OPTIONS="\ + -D BUILD_BITNESS=64 \ + -D CMAKE_BUILD_TYPE=Debug \ + -D CMAKE_INSTALL_INCLUDEDIR=include \ + -D CMAKE_INSTALL_LIBDIR=lib64 \ + -D CMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_PATH}" + +# :: Build GoogleTest ::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +cmake -B "${DIR_SRCS_EXT}/googletest/cmake.bld" \ + -S "${DIR_SRCS_EXT}/googletest" ${CMAKE_OPTIONS} \ + -DCMAKE_INSTALL_PREFIX=/opt/bb +cmake --build "${DIR_SRCS_EXT}/googletest/cmake.bld" -j${PARALLELISM} +cmake --install "${DIR_SRCS_EXT}/googletest/cmake.bld" --prefix "/opt/bb" + + +# :: Build Google Benchmark ::::::::::::::::::::::::::::::::::::::::::::::::::: +cmake -B "${DIR_SRCS_EXT}/google-benchmark/cmake.bld" \ + -S "${DIR_SRCS_EXT}/google-benchmark" ${CMAKE_OPTIONS} \ + -DCMAKE_INSTALL_PREFIX=/opt/bb \ + -DBENCHMARK_DOWNLOAD_DEPENDENCIES="ON" \ + -DBENCHMARK_ENABLE_GTEST_TESTS="false" \ + -DBENCHMARK_ENABLE_TESTING="OFF" +cmake --build "${DIR_SRCS_EXT}/google-benchmark/cmake.bld" -j${PARALLELISM} +cmake --install "${DIR_SRCS_EXT}/google-benchmark/cmake.bld" --prefix "/opt/bb" + +## :: Build zlib :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +# Note: zlib has completely broken CMake install rules, so we must +# specify the install prefix *exactly* as it will be at configure +# time +# https://discourse.cmake.org/t/cmake-install-prefix-not-work/5040 +cmake -B "${DIR_SRCS_EXT}/zlib/cmake.bld" -S "${DIR_SRCS_EXT}/zlib" \ + -D CMAKE_INSTALL_PREFIX="/opt/bb" \ + ${CMAKE_OPTIONS} +# Make and install zlib. +cmake --build "${DIR_SRCS_EXT}/zlib/cmake.bld" -j${PARALLELISM} +cmake --install "${DIR_SRCS_EXT}/zlib/cmake.bld" From b9c5cb4f0bf9bf5b71e18170e4d7c16c150bd79e Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Wed, 10 Jul 2024 16:53:03 +0300 Subject: [PATCH 060/105] build.yaml: debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .../workflows/sanitizers/build_sanitizer.sh | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 681808eb64..f4326df9a5 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -216,3 +216,54 @@ cmake -B "${DIR_SRCS_EXT}/zlib/cmake.bld" -S "${DIR_SRCS_EXT}/zlib" \ # Make and install zlib. cmake --build "${DIR_SRCS_EXT}/zlib/cmake.bld" -j${PARALLELISM} cmake --install "${DIR_SRCS_EXT}/zlib/cmake.bld" + +# :: Build BlazingMQ :::::::::::::::::::::::::::::::::::::::::::::::::::::::::: + +PKG_CONFIG_PATH="/opt/bb/lib64/pkgconfig:/opt/bb/lib/pkgconfig:/opt/bb/share/pkgconfig:$(pkg-config --variable pc_path pkg-config)" \ +cmake -B "${DIR_BUILD_BMQ}" -S "${DIR_SRC_BMQ}" -G Ninja \ + -DBDE_BUILD_TARGET_64=ON \ + -DBDE_BUILD_TARGET_CPP17=ON \ + -DCMAKE_PREFIX_PATH="${DIR_SRCS_EXT}/bde-tools/BdeBuildSystem" \ + -DBDE_BUILD_TARGET_SAFE=1 ${CMAKE_OPTIONS} +cmake --build "${DIR_BUILD_BMQ}" -j${PARALLELISM} \ + --target all.t -v --clean-first + +# :: Create testing scripts ::::::::::::::::::::::::::::::::::::::::::::::::::: +envcfgquery() { + # Parses the '.environment' object from 'sanitizers.json', + # and outputs a string of whitespace-separated 'VAR=VAL' pairs intended to + # be used to set the environment for a command. + # e.g. 'asan' -> 'ASAN_OPTIONS="foo=bar:baz=baf" LSAN_OPTIONS="abc=fgh"' + # + echo $(cfgquery " \ + .${1}.environment | \ + to_entries | \ + map(\"\(.key)=\\\"\(.value | \ + to_entries | \ + map(\"\(.key)=\(.value)\") | \ + join(\":\"))\\\"\") | \ + join(\" \")") | + sed "s|%%SRC%%|$(realpath ${DIR_SRC_BMQ})|g" | + sed "s|%%ROOT%%|$(realpath ${DIR_ROOT})|g" +} + +mkscript() { + local cmd=${1} + local outfile=${2} + + echo '#!/usr/bin/env bash' > ${outfile} + echo "${cmd}" >> ${outfile} + chmod +x ${outfile} +} + +SANITIZER_ENV="BMQ_BUILD=$(realpath ${DIR_BUILD_BMQ}) " +SANITIZER_ENV+="BMQ_REPO=${DIR_SRC_BMQ} " +SANITIZER_ENV+="$(envcfgquery ${SANITIZER_NAME})" + +# 'run-env.sh' runs a command with environment required of the sanitizer. +mkscript "${SANITIZER_ENV} \${@}" "${DIR_BUILD_BMQ}/run-env.sh" + +# 'run-unittests.sh' runs all instrumented unit-tests. +CMD="cd $(realpath ${DIR_SRC_BMQ}) && " +CMD+="${DIR_BUILD_BMQ}/run-env.sh ctest -E mwcsys_executil.t --output-on-failure" +mkscript "${CMD}" "${DIR_BUILD_BMQ}/run-unittests.sh" From c07cb4d05915fc09aab5eb0bbea2ddf2d35a6033 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Wed, 10 Jul 2024 17:33:17 +0300 Subject: [PATCH 061/105] build.yaml: debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers.yaml | 8 +++++--- .github/workflows/sanitizers/build_sanitizer.sh | 4 +++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/sanitizers.yaml b/.github/workflows/sanitizers.yaml index e26cd045f9..2833937494 100644 --- a/.github/workflows/sanitizers.yaml +++ b/.github/workflows/sanitizers.yaml @@ -15,7 +15,7 @@ on: jobs: build_and_run_sanitizer: name: Build and run Sanitizer - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 # defaults: # run: # working-directory: ${{ inputs.working-directory }} @@ -28,11 +28,13 @@ jobs: with: path: deps key: deps-${{ steps.get-hash.outputs.deps_hash }} - - name: List deps/srcs - run: ls -lah ${{ github.workspace }}/deps/srcs # - name: San name # run: echo ${{ inputs.sanitizer-name }} # - name: ROOT name # run: pwd - name: Build BlazingMQ and dependencies with sanitizer instrumentation run: ./.github/workflows/sanitizers/build_sanitizer.sh ${{ inputs.sanitizer-name }} + - name: Run unit tests under sanitizer + run: | + cd ${{ github.workspace }}/build/blazingmq + ./run-unittests.sh diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index f4326df9a5..e2dd9343c3 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -31,7 +31,9 @@ fi SANITIZER_NAME="${1}" -# Uninstall uneeded tools which cause of versions clash +# Github's 'ubuntu-22.04' image contains a lot of preinstalled tools, +# see https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2204-Readme.md. +# Uninstall uneeded tools which cause of versions clash. sudo apt-get purge llvm-14 clang-14 gcc-9 gcc-10 gcc-11 gcc-12 # Install prerequisites From 895f0f9361c8d50d271bbd06a17eaefdb933cff4 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Wed, 10 Jul 2024 18:31:53 +0300 Subject: [PATCH 062/105] Build.yaml: debug build_and_run_asan job Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sanitizers.yaml b/.github/workflows/sanitizers.yaml index 2833937494..18d7d8b1d4 100644 --- a/.github/workflows/sanitizers.yaml +++ b/.github/workflows/sanitizers.yaml @@ -36,5 +36,5 @@ jobs: run: ./.github/workflows/sanitizers/build_sanitizer.sh ${{ inputs.sanitizer-name }} - name: Run unit tests under sanitizer run: | - cd ${{ github.workspace }}/build/blazingmq + cd ${{ github.workspace }}/cmake.bld/Linux ./run-unittests.sh From 81d25deb29c4d2e78a479117ed479a72146354ee Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Thu, 11 Jul 2024 09:29:07 +0300 Subject: [PATCH 063/105] build.yaml: debug asan and msan jobs Signed-off-by: Aleksandr Ivanov --- .github/workflows/build.yaml | 6 ++++++ .github/workflows/sanitizers/build_sanitizer.sh | 6 +++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a875d6f7e1..836b23df2e 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -66,6 +66,12 @@ jobs: sanitizer-name: 'asan' # working-directory: ${{ github.workspace }} + build_and_run_msan: + needs: build_dependencies + uses: ./.github/workflows/sanitizers.yaml + with: + sanitizer-name: 'msan' + # build_and_run_asan: # name: Build and run AddressSanitizer # runs-on: ubuntu-latest diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index e2dd9343c3..d2b5f6c293 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -263,9 +263,9 @@ SANITIZER_ENV+="BMQ_REPO=${DIR_SRC_BMQ} " SANITIZER_ENV+="$(envcfgquery ${SANITIZER_NAME})" # 'run-env.sh' runs a command with environment required of the sanitizer. -mkscript "${SANITIZER_ENV} \${@}" "${DIR_BUILD_BMQ}/run-env.sh" +mkscript "${SANITIZER_ENV} \${@}" "${DIR_BUILD_BMQ}/blazingmq/run-env.sh" # 'run-unittests.sh' runs all instrumented unit-tests. CMD="cd $(realpath ${DIR_SRC_BMQ}) && " -CMD+="${DIR_BUILD_BMQ}/run-env.sh ctest -E mwcsys_executil.t --output-on-failure" -mkscript "${CMD}" "${DIR_BUILD_BMQ}/run-unittests.sh" +CMD+="${DIR_BUILD_BMQ}/blazingmq/run-env.sh ctest -E mwcsys_executil.t --output-on-failure" +mkscript "${CMD}" "${DIR_BUILD_BMQ}/blazingmq/run-unittests.sh" From e7c1b7e226dbe5dbbc5426f4b064728c20bd6ef9 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Thu, 11 Jul 2024 09:38:02 +0300 Subject: [PATCH 064/105] build.yaml: debug asan and msan jobs Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/clang-libcxx-msan.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sanitizers/clang-libcxx-msan.cmake b/.github/workflows/sanitizers/clang-libcxx-msan.cmake index e93abde3b1..2926f0d4c5 100644 --- a/.github/workflows/sanitizers/clang-libcxx-msan.cmake +++ b/.github/workflows/sanitizers/clang-libcxx-msan.cmake @@ -13,7 +13,7 @@ cmake_minimum_required (VERSION 3.25) # endif() # endif() -include("/sanitizers/BBToolchain64.cmake") +include("$ENV{DIR_SCRIPTS}/BBToolchain64.cmake") if(DEFINED ENV{CC}) set(CMAKE_C_COMPILER $ENV{CC} CACHE STRING "Instrumentation C compiler" FORCE) From ef836525ac10924b0a983ae837b4c7c375921240 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Thu, 11 Jul 2024 10:41:09 +0300 Subject: [PATCH 065/105] build.yaml: debug asan and msan jobs Signed-off-by: Aleksandr Ivanov --- .github/workflows/build.yaml | 10 +- .../workflows/sanitizers/build_sanitizer.sh | 190 +----------------- 2 files changed, 12 insertions(+), 188 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 836b23df2e..fe7cea58d2 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -66,11 +66,11 @@ jobs: sanitizer-name: 'asan' # working-directory: ${{ github.workspace }} - build_and_run_msan: - needs: build_dependencies - uses: ./.github/workflows/sanitizers.yaml - with: - sanitizer-name: 'msan' + # build_and_run_msan: + # needs: build_dependencies + # uses: ./.github/workflows/sanitizers.yaml + # with: + # sanitizer-name: 'msan' # build_and_run_asan: # name: Build and run AddressSanitizer diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index d2b5f6c293..77a7090b42 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -22,6 +22,8 @@ set -eux + + # :: Required arguments ::::::::::::::::::::::::::::::::::::::::::::::::::::::: if [ -z ${1} ]; then echo 'Error: Missing sanitizer name.' >&2 @@ -29,34 +31,10 @@ if [ -z ${1} ]; then exit 1 fi -SANITIZER_NAME="${1}" - -# Github's 'ubuntu-22.04' image contains a lot of preinstalled tools, -# see https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2204-Readme.md. -# Uninstall uneeded tools which cause of versions clash. -sudo apt-get purge llvm-14 clang-14 gcc-9 gcc-10 gcc-11 gcc-12 - -# Install prerequisites -sudo apt-get update && sudo apt-get install -qy lsb-release wget software-properties-common gnupg git curl jq ninja-build bison libfl-dev pkg-config -# Install prerequisites for LLVM: latest cmake version, Ubuntu apt repository contains cmake version 3.22.1 -wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | sudo tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null -sudo apt-add-repository -y "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" -sudo apt-get install -qy cmake - -# Install LLVM -wget https://apt.llvm.org/llvm.sh -chmod +x llvm.sh -LLVM_VERSION=18 -sudo ./llvm.sh ${LLVM_VERSION} all +SANITIZER_NAME="${1}" -# Create version-agnostic pointers to required LLVM binaries. -sudo ln -sf /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang -sudo ln -sf /usr/bin/clang++-${LLVM_VERSION} /usr/bin/clang++ -sudo ln -sf /usr/bin/llvm-symbolizer-${LLVM_VERSION} /usr/bin/llvm-symbolizer -# Set some initial constants -PARALLELISM=8 DIR_ROOT="${PWD}" DIR_SCRIPTS="${DIR_ROOT}/.github/workflows/sanitizers" @@ -67,168 +45,11 @@ DIR_BUILD_EXT="${DIR_EXTERNAL}/cmake.bld" DIR_SRC_BMQ="${DIR_ROOT}" DIR_BUILD_BMQ="${DIR_SRC_BMQ}/cmake.bld/Linux" + # Parse sanitizers config cfgquery() { jq "${1}" "${DIR_SCRIPTS}/sanitizers.json" --raw-output } -LLVM_SANITIZER_NAME="$(cfgquery .${SANITIZER_NAME}.llvm_sanitizer_name)" -# Check if llvm specific cmake options are present for the given sanitizer -LLVM_SPECIFIC_CMAKE_OPTIONS="$(cfgquery .${SANITIZER_NAME}.llvm_specific_cmake_options)" -if [[ "$LLVM_SPECIFIC_CMAKE_OPTIONS" == null ]]; then LLVM_SPECIFIC_CMAKE_OPTIONS=""; fi - -# :: checkoutGitRepo() subroutine ::::::::::::::::::::::::::::::::::::::::::::: -checkoutGitRepo() { - local repo=$1 - local ref=$2 - local repoDir=$3 - echo "Checking out ${repo} at ${ref}" - - local repoPath="${DIR_SRCS_EXT}/${repoDir}" - - git clone -b ${ref} ${repo} \ - --depth 1 --single-branch --no-tags -c advice.detachedHead=false "${repoPath}" -} -github_url() { echo "https://github.com/$1.git"; } - -# :: Download external dependencies ::::::::::::::::::::::::::::::: -mkdir -p ${DIR_SRCS_EXT} - -# Download LLVM -LLVM_TAG="llvmorg-18.1.8" -checkoutGitRepo "$(github_url llvm/llvm-project)" "${LLVM_TAG}" "llvm-project" - -# Download google-benchmark -GOOGLE_BENCHMARK_TAG="v1.8.4" -checkoutGitRepo "$(github_url google/benchmark)" "${GOOGLE_BENCHMARK_TAG}" "google-benchmark" - -# Download googletest -GOOGLETEST_TAG="v1.14.0" -checkoutGitRepo "$(github_url google/googletest)" "${GOOGLETEST_TAG}" "googletest" - -# Download zlib -ZLIB_TAG="v1.3.1" -checkoutGitRepo "$(github_url madler/zlib)" "${ZLIB_TAG}" "zlib" - -# :: Build libc++ with required instrumentation ::::::::::::::::::::::::::::::: -# -# The extent to which all dependencies to be compiled with sanitizer-support -# varies by sanitizer. MemorySanitizer is especially unforgiving: Failing to -# link against an instrumented standard library will yield many false -# positives. Concensus is that compiling libc++ with `-fsanitize=memory` is a -# significantly easier endeavor than doing the same with libstdc++ (the gcc -# standard library). -# -# We therefore opt to use libc++ here, just to ensure maximum flexibility. We -# follow build instructions from https://libcxx.llvm.org/BuildingLibcxx.html -LIBCXX_SRC_PATH="${DIR_SRCS_EXT}/llvm-project/runtimes" -LIBCXX_BUILD_PATH="${LIBCXX_SRC_PATH}/cmake.bld" - -cmake -B "${LIBCXX_BUILD_PATH}" \ - -S "${LIBCXX_SRC_PATH}" \ - -DCMAKE_BUILD_TYPE="Debug" \ - -DCMAKE_C_COMPILER="clang" \ - -DCMAKE_CXX_COMPILER="clang++" \ - -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ - -DLLVM_USE_SANITIZER="${LLVM_SANITIZER_NAME}" \ - ${LLVM_SPECIFIC_CMAKE_OPTIONS} - -cmake --build "${LIBCXX_BUILD_PATH}" -j${PARALLELISM} --target cxx cxxabi unwind generate-cxx-headers - -# Variables read by our custom CMake toolchain used to build everything else. -export LIBCXX_BUILD_PATH="$(realpath ${LIBCXX_BUILD_PATH})" -export DIR_SRC_BMQ="${DIR_SRC_BMQ}" -export DIR_SCRIPTS="${DIR_SCRIPTS}" - -# :: Build BDE + NTF :::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -TOOLCHAIN_PATH="${DIR_SCRIPTS}/clang-libcxx-${SANITIZER_NAME}.cmake" -export CC="clang" -export CXX="clang++" -export BBS_BUILD_SYSTEM="ON" -PATH="$PATH:$(realpath ${DIR_SRCS_EXT}/bde-tools/bin)" -export PATH - -pushd ${DIR_SRCS_EXT} - -pushd "bde" -eval "$(bbs_build_env -u dbg_64_safe_cpp20 -b "${DIR_BUILD_EXT}/bde")" -bbs_build configure --toolchain "${TOOLCHAIN_PATH}" -bbs_build build -j${PARALLELISM} -bbs_build --install=/opt/bb --prefix=/ install -popd - -pushd "ntf-core" -# TODO The deprecated flag "-fcoroutines-ts" has been removed in clang -# 17.0.1, but NTF is still using it. We manually change this flag until -# the fix in issue 175307231 is resolved. -sed -i 's/fcoroutines-ts/fcoroutines/g' 'repository.cmake' - -./configure --keep \ - --prefix /opt/bb \ - --output "${DIR_BUILD_EXT}/ntf" \ - --without-warnings-as-errors \ - --without-usage-examples \ - --without-applications \ - --ufid 'dbg_64_safe_cpp20' \ - --toolchain "${TOOLCHAIN_PATH}" -make -j${PARALLELISM} -make install -popd - -# Note: Hack to circumvent faulty behavior in "nts-targets.cmake" -ln -sf "/opt/bb/include" "/opt/include" -ln -sf "/opt/bb/lib64" "/opt/lib64" - -# pushd DIR_SRCS_EXT -popd - -# :: Setup CMake options for all remaining builds ::::::::::::::::::::::::::::: -CMAKE_OPTIONS="\ - -D BUILD_BITNESS=64 \ - -D CMAKE_BUILD_TYPE=Debug \ - -D CMAKE_INSTALL_INCLUDEDIR=include \ - -D CMAKE_INSTALL_LIBDIR=lib64 \ - -D CMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_PATH}" - -# :: Build GoogleTest ::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -cmake -B "${DIR_SRCS_EXT}/googletest/cmake.bld" \ - -S "${DIR_SRCS_EXT}/googletest" ${CMAKE_OPTIONS} \ - -DCMAKE_INSTALL_PREFIX=/opt/bb -cmake --build "${DIR_SRCS_EXT}/googletest/cmake.bld" -j${PARALLELISM} -cmake --install "${DIR_SRCS_EXT}/googletest/cmake.bld" --prefix "/opt/bb" - - -# :: Build Google Benchmark ::::::::::::::::::::::::::::::::::::::::::::::::::: -cmake -B "${DIR_SRCS_EXT}/google-benchmark/cmake.bld" \ - -S "${DIR_SRCS_EXT}/google-benchmark" ${CMAKE_OPTIONS} \ - -DCMAKE_INSTALL_PREFIX=/opt/bb \ - -DBENCHMARK_DOWNLOAD_DEPENDENCIES="ON" \ - -DBENCHMARK_ENABLE_GTEST_TESTS="false" \ - -DBENCHMARK_ENABLE_TESTING="OFF" -cmake --build "${DIR_SRCS_EXT}/google-benchmark/cmake.bld" -j${PARALLELISM} -cmake --install "${DIR_SRCS_EXT}/google-benchmark/cmake.bld" --prefix "/opt/bb" - -## :: Build zlib :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -# Note: zlib has completely broken CMake install rules, so we must -# specify the install prefix *exactly* as it will be at configure -# time -# https://discourse.cmake.org/t/cmake-install-prefix-not-work/5040 -cmake -B "${DIR_SRCS_EXT}/zlib/cmake.bld" -S "${DIR_SRCS_EXT}/zlib" \ - -D CMAKE_INSTALL_PREFIX="/opt/bb" \ - ${CMAKE_OPTIONS} -# Make and install zlib. -cmake --build "${DIR_SRCS_EXT}/zlib/cmake.bld" -j${PARALLELISM} -cmake --install "${DIR_SRCS_EXT}/zlib/cmake.bld" - -# :: Build BlazingMQ :::::::::::::::::::::::::::::::::::::::::::::::::::::::::: - -PKG_CONFIG_PATH="/opt/bb/lib64/pkgconfig:/opt/bb/lib/pkgconfig:/opt/bb/share/pkgconfig:$(pkg-config --variable pc_path pkg-config)" \ -cmake -B "${DIR_BUILD_BMQ}" -S "${DIR_SRC_BMQ}" -G Ninja \ - -DBDE_BUILD_TARGET_64=ON \ - -DBDE_BUILD_TARGET_CPP17=ON \ - -DCMAKE_PREFIX_PATH="${DIR_SRCS_EXT}/bde-tools/BdeBuildSystem" \ - -DBDE_BUILD_TARGET_SAFE=1 ${CMAKE_OPTIONS} -cmake --build "${DIR_BUILD_BMQ}" -j${PARALLELISM} \ - --target all.t -v --clean-first # :: Create testing scripts ::::::::::::::::::::::::::::::::::::::::::::::::::: envcfgquery() { @@ -269,3 +90,6 @@ mkscript "${SANITIZER_ENV} \${@}" "${DIR_BUILD_BMQ}/blazingmq/run-env.sh" CMD="cd $(realpath ${DIR_SRC_BMQ}) && " CMD+="${DIR_BUILD_BMQ}/blazingmq/run-env.sh ctest -E mwcsys_executil.t --output-on-failure" mkscript "${CMD}" "${DIR_BUILD_BMQ}/blazingmq/run-unittests.sh" + +cat "${DIR_BUILD_BMQ}/blazingmq/run-env.sh" +cat "${DIR_BUILD_BMQ}/blazingmq/run-unittests.sh" From d8426f28ea81bc6e61e01da7a13eb7faf780cce4 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Thu, 11 Jul 2024 10:46:06 +0300 Subject: [PATCH 066/105] build.yaml: debug asan and msan jobs Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/build_sanitizer.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 77a7090b42..75428ee6c3 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -35,7 +35,6 @@ fi SANITIZER_NAME="${1}" - DIR_ROOT="${PWD}" DIR_SCRIPTS="${DIR_ROOT}/.github/workflows/sanitizers" DIR_EXTERNAL="${DIR_ROOT}/deps" @@ -46,6 +45,9 @@ DIR_SRC_BMQ="${DIR_ROOT}" DIR_BUILD_BMQ="${DIR_SRC_BMQ}/cmake.bld/Linux" +mkdir -p "${DIR_ROOT}/cmake.bld/Linux/blasingmq" + + # Parse sanitizers config cfgquery() { jq "${1}" "${DIR_SCRIPTS}/sanitizers.json" --raw-output From 21e83ed54d55f1d8fc1446bbc670ca2b4b8c00d5 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Thu, 11 Jul 2024 10:48:17 +0300 Subject: [PATCH 067/105] build.yaml: debug asan and msan jobs Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/build_sanitizer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 75428ee6c3..35f655d18b 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -45,7 +45,7 @@ DIR_SRC_BMQ="${DIR_ROOT}" DIR_BUILD_BMQ="${DIR_SRC_BMQ}/cmake.bld/Linux" -mkdir -p "${DIR_ROOT}/cmake.bld/Linux/blasingmq" +mkdir -p "${DIR_ROOT}/cmake.bld/Linux/blazingmq" # Parse sanitizers config From 04a76357bec2d995130df09350a8f80dfc49e13c Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Thu, 11 Jul 2024 11:58:27 +0300 Subject: [PATCH 068/105] debug asan Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers.yaml | 8 + .../workflows/sanitizers/build_sanitizer.sh | 200 ++++++++++++++++-- 2 files changed, 195 insertions(+), 13 deletions(-) diff --git a/.github/workflows/sanitizers.yaml b/.github/workflows/sanitizers.yaml index 18d7d8b1d4..e634612f3e 100644 --- a/.github/workflows/sanitizers.yaml +++ b/.github/workflows/sanitizers.yaml @@ -34,6 +34,14 @@ jobs: # run: pwd - name: Build BlazingMQ and dependencies with sanitizer instrumentation run: ./.github/workflows/sanitizers/build_sanitizer.sh ${{ inputs.sanitizer-name }} + - name: Cat run-env.sh + run: cat ${{ github.workspace }}/cmake.bld/Linux/run-env.sh + - name: Cat run-unittests.sh + run: cat ${{ github.workspace }}/cmake.bld/Linux/run-unittests.sh + - name: List build + run: ls -lah ${{ github.workspace }}/cmake.bld/Linux + - name: List test binaries + run: find ${{ github.workspace }}/cmake.bld/Linux -name "*.t.tsk" - name: Run unit tests under sanitizer run: | cd ${{ github.workspace }}/cmake.bld/Linux diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 35f655d18b..e2dd9343c3 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -22,8 +22,6 @@ set -eux - - # :: Required arguments ::::::::::::::::::::::::::::::::::::::::::::::::::::::: if [ -z ${1} ]; then echo 'Error: Missing sanitizer name.' >&2 @@ -31,9 +29,34 @@ if [ -z ${1} ]; then exit 1 fi - SANITIZER_NAME="${1}" +# Github's 'ubuntu-22.04' image contains a lot of preinstalled tools, +# see https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2204-Readme.md. +# Uninstall uneeded tools which cause of versions clash. +sudo apt-get purge llvm-14 clang-14 gcc-9 gcc-10 gcc-11 gcc-12 + +# Install prerequisites +sudo apt-get update && sudo apt-get install -qy lsb-release wget software-properties-common gnupg git curl jq ninja-build bison libfl-dev pkg-config + +# Install prerequisites for LLVM: latest cmake version, Ubuntu apt repository contains cmake version 3.22.1 +wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | sudo tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null +sudo apt-add-repository -y "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" +sudo apt-get install -qy cmake + +# Install LLVM +wget https://apt.llvm.org/llvm.sh +chmod +x llvm.sh +LLVM_VERSION=18 +sudo ./llvm.sh ${LLVM_VERSION} all + +# Create version-agnostic pointers to required LLVM binaries. +sudo ln -sf /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang +sudo ln -sf /usr/bin/clang++-${LLVM_VERSION} /usr/bin/clang++ +sudo ln -sf /usr/bin/llvm-symbolizer-${LLVM_VERSION} /usr/bin/llvm-symbolizer + +# Set some initial constants +PARALLELISM=8 DIR_ROOT="${PWD}" DIR_SCRIPTS="${DIR_ROOT}/.github/workflows/sanitizers" @@ -44,14 +67,168 @@ DIR_BUILD_EXT="${DIR_EXTERNAL}/cmake.bld" DIR_SRC_BMQ="${DIR_ROOT}" DIR_BUILD_BMQ="${DIR_SRC_BMQ}/cmake.bld/Linux" - -mkdir -p "${DIR_ROOT}/cmake.bld/Linux/blazingmq" - - # Parse sanitizers config cfgquery() { jq "${1}" "${DIR_SCRIPTS}/sanitizers.json" --raw-output } +LLVM_SANITIZER_NAME="$(cfgquery .${SANITIZER_NAME}.llvm_sanitizer_name)" +# Check if llvm specific cmake options are present for the given sanitizer +LLVM_SPECIFIC_CMAKE_OPTIONS="$(cfgquery .${SANITIZER_NAME}.llvm_specific_cmake_options)" +if [[ "$LLVM_SPECIFIC_CMAKE_OPTIONS" == null ]]; then LLVM_SPECIFIC_CMAKE_OPTIONS=""; fi + +# :: checkoutGitRepo() subroutine ::::::::::::::::::::::::::::::::::::::::::::: +checkoutGitRepo() { + local repo=$1 + local ref=$2 + local repoDir=$3 + echo "Checking out ${repo} at ${ref}" + + local repoPath="${DIR_SRCS_EXT}/${repoDir}" + + git clone -b ${ref} ${repo} \ + --depth 1 --single-branch --no-tags -c advice.detachedHead=false "${repoPath}" +} +github_url() { echo "https://github.com/$1.git"; } + +# :: Download external dependencies ::::::::::::::::::::::::::::::: +mkdir -p ${DIR_SRCS_EXT} + +# Download LLVM +LLVM_TAG="llvmorg-18.1.8" +checkoutGitRepo "$(github_url llvm/llvm-project)" "${LLVM_TAG}" "llvm-project" + +# Download google-benchmark +GOOGLE_BENCHMARK_TAG="v1.8.4" +checkoutGitRepo "$(github_url google/benchmark)" "${GOOGLE_BENCHMARK_TAG}" "google-benchmark" + +# Download googletest +GOOGLETEST_TAG="v1.14.0" +checkoutGitRepo "$(github_url google/googletest)" "${GOOGLETEST_TAG}" "googletest" + +# Download zlib +ZLIB_TAG="v1.3.1" +checkoutGitRepo "$(github_url madler/zlib)" "${ZLIB_TAG}" "zlib" + +# :: Build libc++ with required instrumentation ::::::::::::::::::::::::::::::: +# +# The extent to which all dependencies to be compiled with sanitizer-support +# varies by sanitizer. MemorySanitizer is especially unforgiving: Failing to +# link against an instrumented standard library will yield many false +# positives. Concensus is that compiling libc++ with `-fsanitize=memory` is a +# significantly easier endeavor than doing the same with libstdc++ (the gcc +# standard library). +# +# We therefore opt to use libc++ here, just to ensure maximum flexibility. We +# follow build instructions from https://libcxx.llvm.org/BuildingLibcxx.html +LIBCXX_SRC_PATH="${DIR_SRCS_EXT}/llvm-project/runtimes" +LIBCXX_BUILD_PATH="${LIBCXX_SRC_PATH}/cmake.bld" + +cmake -B "${LIBCXX_BUILD_PATH}" \ + -S "${LIBCXX_SRC_PATH}" \ + -DCMAKE_BUILD_TYPE="Debug" \ + -DCMAKE_C_COMPILER="clang" \ + -DCMAKE_CXX_COMPILER="clang++" \ + -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ + -DLLVM_USE_SANITIZER="${LLVM_SANITIZER_NAME}" \ + ${LLVM_SPECIFIC_CMAKE_OPTIONS} + +cmake --build "${LIBCXX_BUILD_PATH}" -j${PARALLELISM} --target cxx cxxabi unwind generate-cxx-headers + +# Variables read by our custom CMake toolchain used to build everything else. +export LIBCXX_BUILD_PATH="$(realpath ${LIBCXX_BUILD_PATH})" +export DIR_SRC_BMQ="${DIR_SRC_BMQ}" +export DIR_SCRIPTS="${DIR_SCRIPTS}" + +# :: Build BDE + NTF :::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +TOOLCHAIN_PATH="${DIR_SCRIPTS}/clang-libcxx-${SANITIZER_NAME}.cmake" +export CC="clang" +export CXX="clang++" +export BBS_BUILD_SYSTEM="ON" +PATH="$PATH:$(realpath ${DIR_SRCS_EXT}/bde-tools/bin)" +export PATH + +pushd ${DIR_SRCS_EXT} + +pushd "bde" +eval "$(bbs_build_env -u dbg_64_safe_cpp20 -b "${DIR_BUILD_EXT}/bde")" +bbs_build configure --toolchain "${TOOLCHAIN_PATH}" +bbs_build build -j${PARALLELISM} +bbs_build --install=/opt/bb --prefix=/ install +popd + +pushd "ntf-core" +# TODO The deprecated flag "-fcoroutines-ts" has been removed in clang +# 17.0.1, but NTF is still using it. We manually change this flag until +# the fix in issue 175307231 is resolved. +sed -i 's/fcoroutines-ts/fcoroutines/g' 'repository.cmake' + +./configure --keep \ + --prefix /opt/bb \ + --output "${DIR_BUILD_EXT}/ntf" \ + --without-warnings-as-errors \ + --without-usage-examples \ + --without-applications \ + --ufid 'dbg_64_safe_cpp20' \ + --toolchain "${TOOLCHAIN_PATH}" +make -j${PARALLELISM} +make install +popd + +# Note: Hack to circumvent faulty behavior in "nts-targets.cmake" +ln -sf "/opt/bb/include" "/opt/include" +ln -sf "/opt/bb/lib64" "/opt/lib64" + +# pushd DIR_SRCS_EXT +popd + +# :: Setup CMake options for all remaining builds ::::::::::::::::::::::::::::: +CMAKE_OPTIONS="\ + -D BUILD_BITNESS=64 \ + -D CMAKE_BUILD_TYPE=Debug \ + -D CMAKE_INSTALL_INCLUDEDIR=include \ + -D CMAKE_INSTALL_LIBDIR=lib64 \ + -D CMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_PATH}" + +# :: Build GoogleTest ::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +cmake -B "${DIR_SRCS_EXT}/googletest/cmake.bld" \ + -S "${DIR_SRCS_EXT}/googletest" ${CMAKE_OPTIONS} \ + -DCMAKE_INSTALL_PREFIX=/opt/bb +cmake --build "${DIR_SRCS_EXT}/googletest/cmake.bld" -j${PARALLELISM} +cmake --install "${DIR_SRCS_EXT}/googletest/cmake.bld" --prefix "/opt/bb" + + +# :: Build Google Benchmark ::::::::::::::::::::::::::::::::::::::::::::::::::: +cmake -B "${DIR_SRCS_EXT}/google-benchmark/cmake.bld" \ + -S "${DIR_SRCS_EXT}/google-benchmark" ${CMAKE_OPTIONS} \ + -DCMAKE_INSTALL_PREFIX=/opt/bb \ + -DBENCHMARK_DOWNLOAD_DEPENDENCIES="ON" \ + -DBENCHMARK_ENABLE_GTEST_TESTS="false" \ + -DBENCHMARK_ENABLE_TESTING="OFF" +cmake --build "${DIR_SRCS_EXT}/google-benchmark/cmake.bld" -j${PARALLELISM} +cmake --install "${DIR_SRCS_EXT}/google-benchmark/cmake.bld" --prefix "/opt/bb" + +## :: Build zlib :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +# Note: zlib has completely broken CMake install rules, so we must +# specify the install prefix *exactly* as it will be at configure +# time +# https://discourse.cmake.org/t/cmake-install-prefix-not-work/5040 +cmake -B "${DIR_SRCS_EXT}/zlib/cmake.bld" -S "${DIR_SRCS_EXT}/zlib" \ + -D CMAKE_INSTALL_PREFIX="/opt/bb" \ + ${CMAKE_OPTIONS} +# Make and install zlib. +cmake --build "${DIR_SRCS_EXT}/zlib/cmake.bld" -j${PARALLELISM} +cmake --install "${DIR_SRCS_EXT}/zlib/cmake.bld" + +# :: Build BlazingMQ :::::::::::::::::::::::::::::::::::::::::::::::::::::::::: + +PKG_CONFIG_PATH="/opt/bb/lib64/pkgconfig:/opt/bb/lib/pkgconfig:/opt/bb/share/pkgconfig:$(pkg-config --variable pc_path pkg-config)" \ +cmake -B "${DIR_BUILD_BMQ}" -S "${DIR_SRC_BMQ}" -G Ninja \ + -DBDE_BUILD_TARGET_64=ON \ + -DBDE_BUILD_TARGET_CPP17=ON \ + -DCMAKE_PREFIX_PATH="${DIR_SRCS_EXT}/bde-tools/BdeBuildSystem" \ + -DBDE_BUILD_TARGET_SAFE=1 ${CMAKE_OPTIONS} +cmake --build "${DIR_BUILD_BMQ}" -j${PARALLELISM} \ + --target all.t -v --clean-first # :: Create testing scripts ::::::::::::::::::::::::::::::::::::::::::::::::::: envcfgquery() { @@ -86,12 +263,9 @@ SANITIZER_ENV+="BMQ_REPO=${DIR_SRC_BMQ} " SANITIZER_ENV+="$(envcfgquery ${SANITIZER_NAME})" # 'run-env.sh' runs a command with environment required of the sanitizer. -mkscript "${SANITIZER_ENV} \${@}" "${DIR_BUILD_BMQ}/blazingmq/run-env.sh" +mkscript "${SANITIZER_ENV} \${@}" "${DIR_BUILD_BMQ}/run-env.sh" # 'run-unittests.sh' runs all instrumented unit-tests. CMD="cd $(realpath ${DIR_SRC_BMQ}) && " -CMD+="${DIR_BUILD_BMQ}/blazingmq/run-env.sh ctest -E mwcsys_executil.t --output-on-failure" -mkscript "${CMD}" "${DIR_BUILD_BMQ}/blazingmq/run-unittests.sh" - -cat "${DIR_BUILD_BMQ}/blazingmq/run-env.sh" -cat "${DIR_BUILD_BMQ}/blazingmq/run-unittests.sh" +CMD+="${DIR_BUILD_BMQ}/run-env.sh ctest -E mwcsys_executil.t --output-on-failure" +mkscript "${CMD}" "${DIR_BUILD_BMQ}/run-unittests.sh" From 3e6d9538470a8040dd65d7e5db40cfd8b99537fd Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Thu, 11 Jul 2024 13:02:27 +0300 Subject: [PATCH 069/105] debug asan and msan Signed-off-by: Aleksandr Ivanov --- .github/workflows/build.yaml | 11 +++++------ .github/workflows/sanitizers.yaml | 4 +--- .github/workflows/sanitizers/build_sanitizer.sh | 4 ++-- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index fe7cea58d2..c67681960b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -64,13 +64,12 @@ jobs: uses: ./.github/workflows/sanitizers.yaml with: sanitizer-name: 'asan' - # working-directory: ${{ github.workspace }} - # build_and_run_msan: - # needs: build_dependencies - # uses: ./.github/workflows/sanitizers.yaml - # with: - # sanitizer-name: 'msan' + build_and_run_msan: + needs: build_dependencies + uses: ./.github/workflows/sanitizers.yaml + with: + sanitizer-name: 'msan' # build_and_run_asan: # name: Build and run AddressSanitizer diff --git a/.github/workflows/sanitizers.yaml b/.github/workflows/sanitizers.yaml index e634612f3e..d2237e8206 100644 --- a/.github/workflows/sanitizers.yaml +++ b/.github/workflows/sanitizers.yaml @@ -43,6 +43,4 @@ jobs: - name: List test binaries run: find ${{ github.workspace }}/cmake.bld/Linux -name "*.t.tsk" - name: Run unit tests under sanitizer - run: | - cd ${{ github.workspace }}/cmake.bld/Linux - ./run-unittests.sh + run: ${{ github.workspace }}/cmake.bld/Linux/run-unittests.sh diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index e2dd9343c3..6223e56e89 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -266,6 +266,6 @@ SANITIZER_ENV+="$(envcfgquery ${SANITIZER_NAME})" mkscript "${SANITIZER_ENV} \${@}" "${DIR_BUILD_BMQ}/run-env.sh" # 'run-unittests.sh' runs all instrumented unit-tests. -CMD="cd $(realpath ${DIR_SRC_BMQ}) && " -CMD+="${DIR_BUILD_BMQ}/run-env.sh ctest -E mwcsys_executil.t --output-on-failure" +CMD="cd $(realpath ${DIR_BUILD_BMQ}) && " +CMD+="./run-env.sh ctest -E mwcsys_executil.t --output-on-failure" mkscript "${CMD}" "${DIR_BUILD_BMQ}/run-unittests.sh" From 38560d44552b2e0d03de6ceded40f68df8a11cd8 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Thu, 11 Jul 2024 15:00:04 +0300 Subject: [PATCH 070/105] build.yaml: add all sanitizers Signed-off-by: Aleksandr Ivanov --- .github/workflows/build.yaml | 37 ++--- .github/workflows/sanitizers.yaml | 19 --- .../workflows/sanitizers/BBToolchain64.cmake | 127 ++---------------- 3 files changed, 19 insertions(+), 164 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c67681960b..752b9c0ee1 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -71,29 +71,14 @@ jobs: with: sanitizer-name: 'msan' - # build_and_run_asan: - # name: Build and run AddressSanitizer - # runs-on: ubuntu-latest - # needs: build_dependencies - # steps: - # - uses: actions/checkout@v4 - # - name: Get dependencies hash - # id: get-hash - # run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT - # - uses: actions/cache/restore@v4 - # with: - # path: deps - # key: deps-${{ steps.get-hash.outputs.deps_hash }} - # - name: List deps - # run: ls -lah ${{ github.workspace }}/deps - # - name: List deps/srcs - # run: ls -lah ${{ github.workspace }}/deps/srcs - # - name: List deps/srcs/bde - # run: ls -lah ${{ github.workspace }}/deps/srcs/bde - # - name: List deps/srcs/ntf-core - # run: ls -lah ${{ github.workspace }}/deps/srcs/ntf-core - # - name: List deps/_build - # run: ls -lah ${{ github.workspace }}/deps/_build - # - name: List deps - # run: ls -lah ${{ github.workspace }}/deps - \ No newline at end of file + build_and_run_tsan: + needs: build_dependencies + uses: ./.github/workflows/sanitizers.yaml + with: + sanitizer-name: 'tsan' + + build_and_run_ubsan: + needs: build_dependencies + uses: ./.github/workflows/sanitizers.yaml + with: + sanitizer-name: 'ubsan' diff --git a/.github/workflows/sanitizers.yaml b/.github/workflows/sanitizers.yaml index d2237e8206..9f26fc1b5a 100644 --- a/.github/workflows/sanitizers.yaml +++ b/.github/workflows/sanitizers.yaml @@ -7,18 +7,11 @@ on: description: Sanitizer name (asan/msan/tsan/ubsan) type: string required: true - # working-directory: - # description: Working directory to use - # type: string - # default: './' jobs: build_and_run_sanitizer: name: Build and run Sanitizer runs-on: ubuntu-22.04 - # defaults: - # run: - # working-directory: ${{ inputs.working-directory }} steps: - uses: actions/checkout@v4 - name: Get dependencies hash @@ -28,19 +21,7 @@ jobs: with: path: deps key: deps-${{ steps.get-hash.outputs.deps_hash }} - # - name: San name - # run: echo ${{ inputs.sanitizer-name }} - # - name: ROOT name - # run: pwd - name: Build BlazingMQ and dependencies with sanitizer instrumentation run: ./.github/workflows/sanitizers/build_sanitizer.sh ${{ inputs.sanitizer-name }} - - name: Cat run-env.sh - run: cat ${{ github.workspace }}/cmake.bld/Linux/run-env.sh - - name: Cat run-unittests.sh - run: cat ${{ github.workspace }}/cmake.bld/Linux/run-unittests.sh - - name: List build - run: ls -lah ${{ github.workspace }}/cmake.bld/Linux - - name: List test binaries - run: find ${{ github.workspace }}/cmake.bld/Linux -name "*.t.tsk" - name: Run unit tests under sanitizer run: ${{ github.workspace }}/cmake.bld/Linux/run-unittests.sh diff --git a/.github/workflows/sanitizers/BBToolchain64.cmake b/.github/workflows/sanitizers/BBToolchain64.cmake index 7a35b019e3..0636bab823 100644 --- a/.github/workflows/sanitizers/BBToolchain64.cmake +++ b/.github/workflows/sanitizers/BBToolchain64.cmake @@ -1,5 +1,3 @@ -# BBToolchain64.cmake *GENERATED FILE - DO NOT EDIT* -*-cmake-*- - cmake_minimum_required (VERSION 3.22) #@PURPOSE: Configure compiler, ABI flags and directory structure. @@ -9,140 +7,31 @@ cmake_minimum_required (VERSION 3.22) # by using -D or by calling "set" before the top-level CMakeLists.txt call to # project(). **This file should NOT be included directly**. The purpose # of the toolchain file is to prepare the build directory for cross-compiling. -# In our context, this means setting the compiler to be the production ready -# compiler version as well as the flags necessary for ABI compatibility so that -# the artifacts can be used on any machine in production. Search paths for -# CMake find_### methods as well as the CMake PkgConfig integration are -# configured here. In addition, this file will set basic variables necessary -# for navigating both the directory structure. These variables should only -# be referenced by distribution modules such as Pekludge.cmake, as they are not -# guaranteed to exist in alternative toolchains. Extending modules may take -# use of the following variables: -# * DISTRIBUTION_REFROOT - Root of dpkg artifacts. -# * ROBOLIB_REFROOT - Root of classic robolib artifacts. -# * IS_64BIT - Boolean value needed for directory navigation. -# -# Note that both refroot variables will first use the cmake command line -# parameter definition, then the environment variable if set and finally will -# use the toolchain file location to infer the refroots. - -# Set base environment variables. -# if(NOT DEFINED DISTRIBUTION_REFROOT) -# if(DEFINED ENV{DISTRIBUTION_REFROOT}) -# set(DISTRIBUTION_REFROOT "$ENV{DISTRIBUTION_REFROOT}/" CACHE STRING "BB Dpkg root set from environment variable.") -# else() -# get_filename_component(REFROOT ${CMAKE_CURRENT_LIST_DIR}/../../../../ REALPATH) -# set(DISTRIBUTION_REFROOT ${REFROOT}/ CACHE STRING "BB Dpkg root set from toolchain file location.") -# endif() -# endif() -# if(NOT DEFINED ROBOLIB_REFROOT) -# if(DEFINED ENV{ROBOLIB_REFROOT}) -# set(ROBOLIB_REFROOT "$ENV{ROBOLIB_REFROOT}/" CACHE STRING "BB Robo root set from environment variable.") -# else() -# get_filename_component(REFROOT ${CMAKE_CURRENT_LIST_DIR}/../../../../ REALPATH) -# set(ROBOLIB_REFROOT ${REFROOT}/ CACHE STRING "BB Robo root set from toolchain file location.") -# endif() -# endif() -set(IS_64BIT yes CACHE BOOL "Bloomberg tool chain bitness.") -# Set distribution compilers and ABI flags. -# set(CMAKE_C_COMPILER ${DISTRIBUTION_REFROOT}/opt/bb/bin/gcc CACHE FILEPATH "C Compiler path") -# set(CMAKE_CXX_COMPILER ${DISTRIBUTION_REFROOT}/opt/bb/bin/g++ CACHE FILEPATH "C++ Compiler path") -# set(CMAKE_Fortran_COMPILER ${DISTRIBUTION_REFROOT}/opt/bb/bin/gfortran CACHE FILEPATH "Fortran Compiler path") +set(IS_64BIT yes CACHE BOOL "Tool chain bitness.") # Set the compiler flags. Use the CACHE instead of the *_INIT variants # which are modified in the in the compiler initialization modules. -set(CMAKE_C_FLAGS "-march=westmere -m64 -fno-strict-aliasing" CACHE STRING "Bloomberg ABI C flags.") -set(CMAKE_C_FLAGS_RELWITHDEBINFO "-g -O2 -fno-omit-frame-pointer" CACHE STRING "Bloomberg ABI C flags.") -set(CMAKE_CXX_STANDARD 20 CACHE STRING "Bloomberg C++ standard.") -set(CMAKE_CXX_STANDARD_REQUIRED 1 CACHE BOOL "Bloomberg standard version is required.") -set(CMAKE_CXX_FLAGS "-D_GLIBCXX_USE_CXX11_ABI=0 -march=westmere -m64 -fno-strict-aliasing" CACHE STRING "Bloomberg ABI C++ flags.") -set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-g -O2 -fno-omit-frame-pointer" CACHE STRING "Bloomberg ABI C++ flags.") -set(CMAKE_Fortran_FLAGS "-march=westmere -mstackrealign -fasynchronous-unwind-tables -m64 -std=legacy -Wall -Wno-overwrite-recursive -fimplicit-none -fdec -frecursive -fno-automatic -fno-range-check -fno-align-commons -fd-lines-as-comments -static-libgfortran -fno-dec-promotion -fno-dec-old-init -fno-dec-non-integer-index -fno-dec-sequence -fno-dec-add-missing-indexes -fno-dec-non-logical-if -fno-dec-override-kind -fallow-invalid-boz -fallow-argument-mismatch" CACHE STRING "Bloomberg ABI Fortran flags.") -set(CMAKE_Fortran_FLAGS_RELWITHDEBINFO "-g -O2 -fno-omit-frame-pointer" CACHE STRING "Bloomberg ABI Fortran flags.") +set(CMAKE_C_FLAGS "-march=westmere -m64 -fno-strict-aliasing" CACHE STRING "ABI C flags.") +set(CMAKE_C_FLAGS_RELWITHDEBINFO "-g -O2 -fno-omit-frame-pointer" CACHE STRING "ABI C flags.") +set(CMAKE_CXX_STANDARD 20 CACHE STRING "C++ standard.") +set(CMAKE_CXX_STANDARD_REQUIRED 1 CACHE BOOL "Standard version is required.") +set(CMAKE_CXX_FLAGS "-D_GLIBCXX_USE_CXX11_ABI=0 -march=westmere -m64 -fno-strict-aliasing" CACHE STRING "ABI C++ flags.") +set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-g -O2 -fno-omit-frame-pointer" CACHE STRING "ABI C++ flags.") # Compile variable overrides. set(CMAKE_USER_MAKE_RULES_OVERRIDE ${CMAKE_CURRENT_LIST_FILE}) # Sets this file as the override for the builtin compiler variables and rules set(CMAKE_C_LINK_FLAGS "-fasynchronous-unwind-tables -Wl,--enable-new-dtags -Wl,--gc-sections -Wl,-uplink_timestamp___") set(CMAKE_CXX_LINK_FLAGS "-fasynchronous-unwind-tables -Wl,--enable-new-dtags -Wl,--gc-sections -Wl,-uplink_timestamp___") -set(CMAKE_Fortran_LINK_FLAGS "") - -# Ninja generator can preprocess source files on some platforms to build a better dependency tree than the Makefiles generator. -# We don't use include directives in our Fortran here at Bloomberg so this is not needed. -set(CMAKE_Fortran_PREPROCESS OFF) - -set(CMAKE_Fortran_IMPLICIT_LINK_LIBRARIES "-l:libgfortran.a;-l:libquadmath.a") - -# Set the platform RPATH for production deployment. -set(CMAKE_PLATFORM_REQUIRED_RUNTIME_PATH /usr/lib64 /lib64 /bb/bin/so/64/NS02 /bb/bin/so/64 /bb/bin .) - -# Set the local module path to include standard Bloomberg modules. -list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR} ${DISTRIBUTION_REFROOT}/opt/bb/share/cmake/Modules) - -# Set the program path for pkg-config. Versions 0.29.1 and older without our -# patch must be avoided since they do not scale to the size of our distribution. -# find_program(PKG_CONFIG_EXECUTABLE pkg-config PATHS -# ${DISTRIBUTION_REFROOT}/opt/bb/lib64/bin -# /opt/bb/lib64/bin -# NO_SYSTEM_ENVIRONMENT_PATH) - -# Toolchain files could be invoked multiple times in a single CMake run, -# therefore we need to check if we have appended the refroot pkgconfig path -# in a previous invokation before appending. We cannot use a cached variable -# to achieve this because the cache is not available between separate -# invokations of the toolchain file. -# set(ROBO_PKG_CONFIG_PATH "${DISTRIBUTION_REFROOT}/opt/bb/lib64/robo/pkgconfig:${DISTRIBUTION_REFROOT}/opt/bb/lib64/pkgconfig" CACHE STRING "The location of the robo pkgconfig files.") -# if (DEFINED ENV{PKG_CONFIG_PATH} AND NOT "$ENV{PKG_CONFIG_PATH}" MATCHES ".*${ROBO_PKG_CONFIG_PATH}$") -# message(STATUS "WARNING: Using user supplied PKG_CONFIG_PATH=$ENV{PKG_CONFIG_PATH}") -# endif() -# if (NOT "$ENV{PKG_CONFIG_PATH}" MATCHES ".*${ROBO_PKG_CONFIG_PATH}$") -# set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:${ROBO_PKG_CONFIG_PATH}") -# endif() - -# set(ENV{PKG_CONFIG_SYSROOT_DIR} ${DISTRIBUTION_REFROOT}) # Set the path for looking up includes, libs and files. # list(APPEND CMAKE_SYSTEM_PREFIX_PATH ${DISTRIBUTION_REFROOT}/opt/bb) set_property(GLOBAL PROPERTY FIND_LIBRARY_USE_LIB64_PATHS yes) -# Set the installation directory for libraries. This is for use as the +# Set the installation directory for libraries. This is for use as the # DESTINATION for calls to the cmake 'install' command. Note that this is not # an actual cmake variable, but has become a de facto standard for use in # systems that employ a lib/lib64 deployment architecture. To allow for use # without a toolchain, use "./${CMAKE_INSTALL_LIBDIR}" to avoid configuration # errors when this variable is not set. set(CMAKE_INSTALL_LIBDIR lib64) - -# if ("amd64" STREQUAL "aix6-powerpc" AND IS_64BIT) -# # On AIX, the archiver defaults to 32 bit only. This is a known issue -# # in CMake and unlikely to be fixed. The following is the recommended -# # workaround. Note that the AIX archiver can also use an environment -# # variable, but in cmake, there is no way to set the env at build time. -# set(CMAKE_CXX_ARCHIVE_CREATE -# " -X64 cr ") -# set(CMAKE_CXX_ARCHIVE_APPEND -# " -X64 r ") -# set(CMAKE_CXX_ARCHIVE_FINISH -# " -X64 ") -# set(CMAKE_C_ARCHIVE_CREATE -# " -X64 cr ") -# set(CMAKE_C_ARCHIVE_APPEND -# " -X64 r ") -# set(CMAKE_C_ARCHIVE_FINISH -# " -X64 ") -# set(CMAKE_Fortran_ARCHIVE_CREATE -# " -X64 cr ") -# set(CMAKE_Fortran_ARCHIVE_APPEND -# " -X64 r ") -# set(CMAKE_Fortran_ARCHIVE_FINISH -# " -X64 ") -# endif() - -# ----------------------------------------------------------------------------- -# NOTICE: -# Copyright (C) Bloomberg L.P., 2016 -# All Rights Reserved. -# Property of Bloomberg L.P. (BLP) -# This software is made available solely pursuant to the -# terms of a BLP license agreement which governs its use. -# ----------------------------- END-OF-FILE ----------------------------------- From 778c2e7679a719c94e494fb2bfa337cdd2272c0f Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Thu, 11 Jul 2024 15:16:09 +0300 Subject: [PATCH 071/105] add cmake toolchain files for tsan and ubsan Signed-off-by: Aleksandr Ivanov --- .../sanitizers/clang-libcxx-tsan.cmake | 117 ++++++++++++++++++ .../sanitizers/clang-libcxx-ubsan.cmake | 115 +++++++++++++++++ 2 files changed, 232 insertions(+) create mode 100644 .github/workflows/sanitizers/clang-libcxx-tsan.cmake create mode 100644 .github/workflows/sanitizers/clang-libcxx-ubsan.cmake diff --git a/.github/workflows/sanitizers/clang-libcxx-tsan.cmake b/.github/workflows/sanitizers/clang-libcxx-tsan.cmake new file mode 100644 index 0000000000..9ce5d05207 --- /dev/null +++ b/.github/workflows/sanitizers/clang-libcxx-tsan.cmake @@ -0,0 +1,117 @@ +# Compiler-less toolchain for building with {Clang, libc++} + ThreadSanitizer. +# The actual compiler is passed via `CXX` and `CC` environment variables. +#-=- + +cmake_minimum_required (VERSION 3.25) + +# if(NOT DEFINED DISTRIBUTION_REFROOT) +# if(DEFINED ENV{DISTRIBUTION_REFROOT}) +# set(DISTRIBUTION_REFROOT "$ENV{DISTRIBUTION_REFROOT}/" CACHE STRING "BB Dpkg root set from environment variable.") +# else() +# get_filename_component(REFROOT ${CMAKE_CURRENT_LIST_DIR}/../../../../../ REALPATH) +# set(DISTRIBUTION_REFROOT ${REFROOT}/ CACHE STRING "BB Dpkg root set from toolchain file location.") +# endif() +# endif() + +include("$ENV{DIR_SCRIPTS}/BBToolchain64.cmake") + +if(DEFINED ENV{CC}) + set(CMAKE_C_COMPILER $ENV{CC} CACHE STRING "Instrumentation C compiler" FORCE) +endif() + +if(DEFINED ENV{CXX}) + set(CMAKE_CXX_COMPILER $ENV{CXX} CACHE STRING "Instrumentation C++ compiler" FORCE) +endif() + +set(LIBCXX_BUILD_PATH "$ENV{LIBCXX_BUILD_PATH}") + +# Force disabling the use of Readline. This is a holdover until readline builds with -fPIC. +# https://bbgithub.dev.bloomberg.com/swt/readline/issues/8 +set(BMQ_DISABLE_READLINE TRUE) + +set(TOOLCHAIN_CXX_FLAGS "${CMAKE_CXX_FLAGS_DEBUG}") +set(TOOLCHAIN_C_FLAGS "${CMAKE_C_FLAGS_DEBUG}") + +# Build shared flags. +string(CONCAT TOOLCHAIN_SHARED_FLAGS + "${}" + "${TOOLCHAIN_SHARED_FLAGS} " + "-O0 " + "-g " + "-fno-omit-frame-pointer " + "-fno-optimize-sibling-calls " + "-fdiagnostics-show-option " + "-isystem/usr/include " + "-isystem/usr/lib/llvm-18/lib/clang/18/include " + ) + +# Apply shared flags to each language. +string(CONCAT TOOLCHAIN_CXX_FLAGS + "${TOOLCHAIN_CXX_FLAGS} " + "${TOOLCHAIN_SHARED_FLAGS} " + ) +string(CONCAT TOOLCHAIN_C_FLAGS + "${TOOLCHAIN_C_FLAGS} " + "${TOOLCHAIN_SHARED_FLAGS} " + ) + +# Use libc++ standard library for C++. +string(CONCAT TOOLCHAIN_CXX_FLAGS + "${TOOLCHAIN_CXX_FLAGS} " + "-stdlib=libc++ " + "-I${LIBCXX_BUILD_PATH}/include/c++/v1 " + ) + +# Suppress some warnings when building C++ code. +string(CONCAT TOOLCHAIN_CXX_FLAGS + "${TOOLCHAIN_CXX_FLAGS} " + "-Wno-c++98-compat " + "-Wno-c++98-compat-extra-semi " + "-Wno-c++98-compat-pedantic " + "-Wno-deprecated " + "-Wno-deprecated-declarations " + "-Wno-disabled-macro-expansion " + "-Wno-extra-semi-stmt " + "-Wno-inconsistent-missing-destructor-override " + "-Wno-inconsistent-missing-override " + "-Wno-old-style-cast " + "-Wno-undef " + "-Wno-zero-as-null-pointer-constant " + ) + +# Define linker flags (used for both shared-objects and executables). +string( CONCAT TOOLCHAIN_LINKER_FLAGS + "${CMAKE_LINKER_FLAGS_DEBUG}" + "-stdlib=libc++ " + "-L${LIBCXX_BUILD_PATH}/lib " + "-Wl,-rpath,${LIBCXX_BUILD_PATH}/lib " + "-lc++abi " + ) + +set( TOOLCHAIN_EXE_FLAGS "${TOOLCHAIN_LINKER_FLAGS}" ) + +macro(set_build_type type) + set(CMAKE_CXX_FLAGS_${type} + "${TOOLCHAIN_CXX_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) + set(CMAKE_C_FLAGS_${type} + "${TOOLCHAIN_C_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) + set(CMAKE_EXE_LINKER_FLAGS_${type} + "${TOOLCHAIN_EXE_FLAGS} ${TOOLCHAIN_${type}_FLAGS} -static-libsan" CACHE STRING "Default" FORCE) + set(CMAKE_SHARED_LINKER_FLAGS_${type} + "${TOOLCHAIN_LINKER_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) + set(CMAKE_MODULE_LINKER_FLAGS_${type} + "${TOOLCHAIN_LINKER_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) +endmacro() + +set(TSAN_SUPPRESSION_LIST_PATH "$ENV{DIR_SRC_BMQ}/etc/tsansup.txt") +set(THREAD_SANITIZER TRUE) + +set( TOOLCHAIN_DEBUG_FLAGS + "-fsanitize=thread " + ) + +# Set the final configuration variables, as understood by CMake. +set_build_type(DEBUG) + +# Disable GNU c++ extensions. +set(CMAKE_CXX_EXTENSIONS OFF) diff --git a/.github/workflows/sanitizers/clang-libcxx-ubsan.cmake b/.github/workflows/sanitizers/clang-libcxx-ubsan.cmake new file mode 100644 index 0000000000..664f4b26da --- /dev/null +++ b/.github/workflows/sanitizers/clang-libcxx-ubsan.cmake @@ -0,0 +1,115 @@ +# Compiler-less toolchain for building with {Clang, libc++} + AddressSanitizer. +# The actual compiler is passed via `CXX` and `CC` environment variables. +#-=- + +cmake_minimum_required (VERSION 3.25) + +# if(NOT DEFINED DISTRIBUTION_REFROOT) +# if(DEFINED ENV{DISTRIBUTION_REFROOT}) +# set(DISTRIBUTION_REFROOT "$ENV{DISTRIBUTION_REFROOT}/" CACHE STRING "BB Dpkg root set from environment variable.") +# else() +# get_filename_component(REFROOT ${CMAKE_CURRENT_LIST_DIR}/../../../../../ REALPATH) +# set(DISTRIBUTION_REFROOT ${REFROOT}/ CACHE STRING "BB Dpkg root set from toolchain file location.") +# endif() +# endif() + +include("$ENV{DIR_SCRIPTS}/BBToolchain64.cmake") + +if(DEFINED ENV{CC}) + set(CMAKE_C_COMPILER $ENV{CC} CACHE STRING "Instrumentation C compiler" FORCE) +endif() + +if(DEFINED ENV{CXX}) + set(CMAKE_CXX_COMPILER $ENV{CXX} CACHE STRING "Instrumentation C++ compiler" FORCE) +endif() + +set(LIBCXX_BUILD_PATH "$ENV{LIBCXX_BUILD_PATH}") + +# Force disabling the use of Readline. This is a holdover until readline builds with -fPIC. +# https://bbgithub.dev.bloomberg.com/swt/readline/issues/8 +set(BMQ_DISABLE_READLINE TRUE) + +set(TOOLCHAIN_CXX_FLAGS "${CMAKE_CXX_FLAGS_DEBUG}") +set(TOOLCHAIN_C_FLAGS "${CMAKE_C_FLAGS_DEBUG}") + +# Build shared flags. +string(CONCAT TOOLCHAIN_SHARED_FLAGS + "${}" + "${TOOLCHAIN_SHARED_FLAGS} " + "-O0 " + "-g " + "-fno-omit-frame-pointer " + "-fno-optimize-sibling-calls " + "-fdiagnostics-show-option " + "-isystem/usr/include " + "-isystem/usr/lib/llvm-18/lib/clang/18/include " + ) + +# Apply shared flags to each language. +string(CONCAT TOOLCHAIN_CXX_FLAGS + "${TOOLCHAIN_CXX_FLAGS} " + "${TOOLCHAIN_SHARED_FLAGS} " + ) +string(CONCAT TOOLCHAIN_C_FLAGS + "${TOOLCHAIN_C_FLAGS} " + "${TOOLCHAIN_SHARED_FLAGS} " + ) + +# Use libc++ standard library for C++. +string(CONCAT TOOLCHAIN_CXX_FLAGS + "${TOOLCHAIN_CXX_FLAGS} " + "-stdlib=libc++ " + "-I${LIBCXX_BUILD_PATH}/include/c++/v1 " + ) + +# Suppress some warnings when building C++ code. +string(CONCAT TOOLCHAIN_CXX_FLAGS + "${TOOLCHAIN_CXX_FLAGS} " + "-Wno-c++98-compat " + "-Wno-c++98-compat-extra-semi " + "-Wno-c++98-compat-pedantic " + "-Wno-deprecated " + "-Wno-deprecated-declarations " + "-Wno-disabled-macro-expansion " + "-Wno-extra-semi-stmt " + "-Wno-inconsistent-missing-destructor-override " + "-Wno-inconsistent-missing-override " + "-Wno-old-style-cast " + "-Wno-undef " + "-Wno-zero-as-null-pointer-constant " + ) + +# Define linker flags (used for both shared-objects and executables). +string( CONCAT TOOLCHAIN_LINKER_FLAGS + "${CMAKE_LINKER_FLAGS_DEBUG}" + "-stdlib=libc++ " + "-L${LIBCXX_BUILD_PATH}/lib " + "-Wl,-rpath,${LIBCXX_BUILD_PATH}/lib " + "-lc++abi " + ) + +set( TOOLCHAIN_EXE_FLAGS "${TOOLCHAIN_LINKER_FLAGS}" ) + +macro(set_build_type type) + set(CMAKE_CXX_FLAGS_${type} + "${TOOLCHAIN_CXX_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) + set(CMAKE_C_FLAGS_${type} + "${TOOLCHAIN_C_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) + set(CMAKE_EXE_LINKER_FLAGS_${type} + "${TOOLCHAIN_EXE_FLAGS} ${TOOLCHAIN_${type}_FLAGS} -static-libsan" CACHE STRING "Default" FORCE) + set(CMAKE_SHARED_LINKER_FLAGS_${type} + "${TOOLCHAIN_LINKER_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) + set(CMAKE_MODULE_LINKER_FLAGS_${type} + "${TOOLCHAIN_LINKER_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) +endmacro() + +# Add MemorySanitizer-related flags. +set( TOOLCHAIN_DEBUG_FLAGS + "-fsanitize=undefined " + ) + +# Set the final configuration variables, as understood by CMake. +set_build_type(DEBUG) + +# Disable GNU c++ extensions. +set(CMAKE_CXX_EXTENSIONS OFF) From fd01763794c89471ded89253a6545e2e5d669bbe Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Fri, 12 Jul 2024 12:05:19 +0300 Subject: [PATCH 072/105] Cleanup scripts Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/Dockerfile | 6 - .github/workflows/sanitizers/build_deps.sh | 84 ------ .../workflows/sanitizers/build_sanitizer.sh | 56 ++-- .../sanitizers/clang-libcxx-asan.cmake | 18 +- .../sanitizers/clang-libcxx-msan.cmake | 18 +- .../sanitizers/clang-libcxx-tsan.cmake | 18 +- .../sanitizers/clang-libcxx-ubsan.cmake | 18 +- .github/workflows/sanitizers/script.sh | 251 ------------------ ...{BBToolchain64.cmake => toolchain64.cmake} | 10 +- 9 files changed, 58 insertions(+), 421 deletions(-) delete mode 100644 .github/workflows/sanitizers/Dockerfile delete mode 100644 .github/workflows/sanitizers/build_deps.sh delete mode 100644 .github/workflows/sanitizers/script.sh rename .github/workflows/sanitizers/{BBToolchain64.cmake => toolchain64.cmake} (90%) diff --git a/.github/workflows/sanitizers/Dockerfile b/.github/workflows/sanitizers/Dockerfile deleted file mode 100644 index 0d11d6012f..0000000000 --- a/.github/workflows/sanitizers/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -FROM ubuntu:jammy-20240627.1 - -#RUN apt update \ -# && apt install -y build-essential cmake clang-format clang-tidy lcov libbenchmark-dev libclang-dev libgoogle-perftools-dev python3 python3-pip python-is-python3 \ -# && rm -rf /var/lib/apt/lists/* /var/log/dpkg.log - diff --git a/.github/workflows/sanitizers/build_deps.sh b/.github/workflows/sanitizers/build_deps.sh deleted file mode 100644 index b256d9d288..0000000000 --- a/.github/workflows/sanitizers/build_deps.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash - -# This script downloads, builds, and installs the required build dependencies of BMQ -# from github.com/bloomberg. Software packages are installed to the /opt/bb prefix. - -set -euxo pipefail - -if [ $# == 1 ]; then - if [[ $1 == "only-download" ]]; then - DO_BUILD=false - else - echo "Unexpected optional argument, only 'only-download' is supported" - exit 1 - fi -else - DO_BUILD=true -fi - -fetch_git() { - local org=$1 - local repo=$2 - mkdir -p srcs - - if [ -d "srcs/${repo}" ]; then - return 0 - fi - - if [ -z "${3:-}" ] - then - # Clone the latest 'main' branch if no specific release tag provided - local branch="main" - curl -SL "https://github.com/${org}/${repo}/archive/refs/heads/${branch}.tar.gz" | tar -xzC srcs/ - mv "srcs/${repo}-${branch}" "srcs/${repo}" - else - local tag=$3 - curl -SL "https://github.com/${org}/${repo}/archive/refs/tags/${tag}.tar.gz" | tar -xzC srcs/ - mv "srcs/${repo}-${tag}" "srcs/${repo}" - fi -} - -fetch_deps() { - fetch_git bloomberg bde-tools 4.8.0.0 - fetch_git bloomberg bde 4.8.0.0 - fetch_git bloomberg ntf-core 2.4.2 -} - -configure() { - PATH="$PATH:$(realpath srcs/bde-tools/bin)" - export PATH - eval "$(bbs_build_env -u opt_64_cpp17)" -} - -build_bde() { - pushd srcs/bde - bbs_build configure - bbs_build build -j8 - bbs_build --install=/opt/bb --prefix=/ install - popd -} - -build_ntf() { - pushd srcs/ntf-core - ./configure \ - --keep \ - --prefix /opt/bb \ - --without-usage-examples \ - --without-applications \ - --without-warnings-as-errors \ - --ufid opt_64_cpp17 - make -j8 - make install - popd -} - -build() { - build_bde - build_ntf -} - -fetch_deps -configure -if [ "${DO_BUILD}" = true ]; then - build -fi diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 6223e56e89..7d8b0232a4 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -18,7 +18,7 @@ # 6) Build sanitizer-instrumented BlazingMQ unit tests. # 7) Generate scripts to run unit tests: # ./cmake.bld/Linux/run-unittests.sh -# This script is used as-is by CI to run unit tests with sanitizer. +# This script is used as-is by CI to run unit tests under sanitizer. set -eux @@ -34,13 +34,32 @@ SANITIZER_NAME="${1}" # Github's 'ubuntu-22.04' image contains a lot of preinstalled tools, # see https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2204-Readme.md. # Uninstall uneeded tools which cause of versions clash. -sudo apt-get purge llvm-14 clang-14 gcc-9 gcc-10 gcc-11 gcc-12 +sudo apt-get purge \ + llvm-14 \ + clang-14 \ + gcc-9 \ + gcc-10 \ + gcc-11 \ + gcc-12 # Install prerequisites -sudo apt-get update && sudo apt-get install -qy lsb-release wget software-properties-common gnupg git curl jq ninja-build bison libfl-dev pkg-config +sudo apt-get update && sudo apt-get install -qy \ + lsb-release \ + wget \ + software-properties-common \ + gnupg \ + git \ + curl \ + jq \ + ninja-build \ + bison \ + libfl-dev \ + pkg-config # Install prerequisites for LLVM: latest cmake version, Ubuntu apt repository contains cmake version 3.22.1 -wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | sudo tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null +wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null \ + | gpg --dearmor - \ + | sudo tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null sudo apt-add-repository -y "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" sudo apt-get install -qy cmake @@ -76,7 +95,6 @@ LLVM_SANITIZER_NAME="$(cfgquery .${SANITIZER_NAME}.llvm_sanitizer_name)" LLVM_SPECIFIC_CMAKE_OPTIONS="$(cfgquery .${SANITIZER_NAME}.llvm_specific_cmake_options)" if [[ "$LLVM_SPECIFIC_CMAKE_OPTIONS" == null ]]; then LLVM_SPECIFIC_CMAKE_OPTIONS=""; fi -# :: checkoutGitRepo() subroutine ::::::::::::::::::::::::::::::::::::::::::::: checkoutGitRepo() { local repo=$1 local ref=$2 @@ -90,7 +108,7 @@ checkoutGitRepo() { } github_url() { echo "https://github.com/$1.git"; } -# :: Download external dependencies ::::::::::::::::::::::::::::::: +# Download external dependencies mkdir -p ${DIR_SRCS_EXT} # Download LLVM @@ -109,7 +127,7 @@ checkoutGitRepo "$(github_url google/googletest)" "${GOOGLETEST_TAG}" "googletes ZLIB_TAG="v1.3.1" checkoutGitRepo "$(github_url madler/zlib)" "${ZLIB_TAG}" "zlib" -# :: Build libc++ with required instrumentation ::::::::::::::::::::::::::::::: +# Build libc++ with required instrumentation # # The extent to which all dependencies to be compiled with sanitizer-support # varies by sanitizer. MemorySanitizer is especially unforgiving: Failing to @@ -139,7 +157,6 @@ export LIBCXX_BUILD_PATH="$(realpath ${LIBCXX_BUILD_PATH})" export DIR_SRC_BMQ="${DIR_SRC_BMQ}" export DIR_SCRIPTS="${DIR_SCRIPTS}" -# :: Build BDE + NTF :::::::::::::::::::::::::::::::::::::::::::::::::::::::::: TOOLCHAIN_PATH="${DIR_SCRIPTS}/clang-libcxx-${SANITIZER_NAME}.cmake" export CC="clang" export CXX="clang++" @@ -147,16 +164,15 @@ export BBS_BUILD_SYSTEM="ON" PATH="$PATH:$(realpath ${DIR_SRCS_EXT}/bde-tools/bin)" export PATH -pushd ${DIR_SRCS_EXT} - -pushd "bde" +# Build BDE + NTF +pushd "${DIR_SRCS_EXT}/bde" eval "$(bbs_build_env -u dbg_64_safe_cpp20 -b "${DIR_BUILD_EXT}/bde")" bbs_build configure --toolchain "${TOOLCHAIN_PATH}" bbs_build build -j${PARALLELISM} bbs_build --install=/opt/bb --prefix=/ install popd -pushd "ntf-core" +pushd "${DIR_SRCS_EXT}/ntf-core" # TODO The deprecated flag "-fcoroutines-ts" has been removed in clang # 17.0.1, but NTF is still using it. We manually change this flag until # the fix in issue 175307231 is resolved. @@ -178,10 +194,7 @@ popd ln -sf "/opt/bb/include" "/opt/include" ln -sf "/opt/bb/lib64" "/opt/lib64" -# pushd DIR_SRCS_EXT -popd - -# :: Setup CMake options for all remaining builds ::::::::::::::::::::::::::::: +# Setup CMake options for all remaining builds CMAKE_OPTIONS="\ -D BUILD_BITNESS=64 \ -D CMAKE_BUILD_TYPE=Debug \ @@ -189,7 +202,7 @@ CMAKE_OPTIONS="\ -D CMAKE_INSTALL_LIBDIR=lib64 \ -D CMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_PATH}" -# :: Build GoogleTest ::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +# Build GoogleTest cmake -B "${DIR_SRCS_EXT}/googletest/cmake.bld" \ -S "${DIR_SRCS_EXT}/googletest" ${CMAKE_OPTIONS} \ -DCMAKE_INSTALL_PREFIX=/opt/bb @@ -197,7 +210,7 @@ cmake --build "${DIR_SRCS_EXT}/googletest/cmake.bld" -j${PARALLELISM} cmake --install "${DIR_SRCS_EXT}/googletest/cmake.bld" --prefix "/opt/bb" -# :: Build Google Benchmark ::::::::::::::::::::::::::::::::::::::::::::::::::: +# Build Google Benchmark cmake -B "${DIR_SRCS_EXT}/google-benchmark/cmake.bld" \ -S "${DIR_SRCS_EXT}/google-benchmark" ${CMAKE_OPTIONS} \ -DCMAKE_INSTALL_PREFIX=/opt/bb \ @@ -207,7 +220,7 @@ cmake -B "${DIR_SRCS_EXT}/google-benchmark/cmake.bld" \ cmake --build "${DIR_SRCS_EXT}/google-benchmark/cmake.bld" -j${PARALLELISM} cmake --install "${DIR_SRCS_EXT}/google-benchmark/cmake.bld" --prefix "/opt/bb" -## :: Build zlib :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +# Build zlib # Note: zlib has completely broken CMake install rules, so we must # specify the install prefix *exactly* as it will be at configure # time @@ -219,8 +232,7 @@ cmake -B "${DIR_SRCS_EXT}/zlib/cmake.bld" -S "${DIR_SRCS_EXT}/zlib" \ cmake --build "${DIR_SRCS_EXT}/zlib/cmake.bld" -j${PARALLELISM} cmake --install "${DIR_SRCS_EXT}/zlib/cmake.bld" -# :: Build BlazingMQ :::::::::::::::::::::::::::::::::::::::::::::::::::::::::: - +# Build BlazingMQ PKG_CONFIG_PATH="/opt/bb/lib64/pkgconfig:/opt/bb/lib/pkgconfig:/opt/bb/share/pkgconfig:$(pkg-config --variable pc_path pkg-config)" \ cmake -B "${DIR_BUILD_BMQ}" -S "${DIR_SRC_BMQ}" -G Ninja \ -DBDE_BUILD_TARGET_64=ON \ @@ -230,7 +242,7 @@ cmake -B "${DIR_BUILD_BMQ}" -S "${DIR_SRC_BMQ}" -G Ninja \ cmake --build "${DIR_BUILD_BMQ}" -j${PARALLELISM} \ --target all.t -v --clean-first -# :: Create testing scripts ::::::::::::::::::::::::::::::::::::::::::::::::::: +# Create testing script envcfgquery() { # Parses the '.environment' object from 'sanitizers.json', # and outputs a string of whitespace-separated 'VAR=VAL' pairs intended to diff --git a/.github/workflows/sanitizers/clang-libcxx-asan.cmake b/.github/workflows/sanitizers/clang-libcxx-asan.cmake index 580637eec4..853ab64d91 100644 --- a/.github/workflows/sanitizers/clang-libcxx-asan.cmake +++ b/.github/workflows/sanitizers/clang-libcxx-asan.cmake @@ -1,19 +1,9 @@ # Compiler-less toolchain for building with {Clang, libc++} + AddressSanitizer. # The actual compiler is passed via `CXX` and `CC` environment variables. -#-=- cmake_minimum_required (VERSION 3.25) -# if(NOT DEFINED DISTRIBUTION_REFROOT) -# if(DEFINED ENV{DISTRIBUTION_REFROOT}) -# set(DISTRIBUTION_REFROOT "$ENV{DISTRIBUTION_REFROOT}/" CACHE STRING "BB Dpkg root set from environment variable.") -# else() -# get_filename_component(REFROOT ${CMAKE_CURRENT_LIST_DIR}/../../../../../ REALPATH) -# set(DISTRIBUTION_REFROOT ${REFROOT}/ CACHE STRING "BB Dpkg root set from toolchain file location.") -# endif() -# endif() - -include("$ENV{DIR_SCRIPTS}/BBToolchain64.cmake") +include("$ENV{DIR_SCRIPTS}/toolchain64.cmake") if(DEFINED ENV{CC}) set(CMAKE_C_COMPILER $ENV{CC} CACHE STRING "Instrumentation C compiler" FORCE) @@ -23,6 +13,10 @@ if(DEFINED ENV{CXX}) set(CMAKE_CXX_COMPILER $ENV{CXX} CACHE STRING "Instrumentation C++ compiler" FORCE) endif() +if(DEFINED ENV{CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES}) + set(CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES $ENV{CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES}) +endif() + set(LIBCXX_BUILD_PATH "$ENV{LIBCXX_BUILD_PATH}") # Force disabling the use of Readline. This is a holdover until readline builds with -fPIC. @@ -41,8 +35,6 @@ string(CONCAT TOOLCHAIN_SHARED_FLAGS "-fno-omit-frame-pointer " "-fno-optimize-sibling-calls " "-fdiagnostics-show-option " - "-isystem/usr/include " - "-isystem/usr/lib/llvm-18/lib/clang/18/include " ) # Apply shared flags to each language. diff --git a/.github/workflows/sanitizers/clang-libcxx-msan.cmake b/.github/workflows/sanitizers/clang-libcxx-msan.cmake index 2926f0d4c5..a8cffa36cb 100644 --- a/.github/workflows/sanitizers/clang-libcxx-msan.cmake +++ b/.github/workflows/sanitizers/clang-libcxx-msan.cmake @@ -1,19 +1,9 @@ # Compiler-less toolchain for building with {Clang, libc++} + MemorySanitizer. # The actual compiler is passed via `CXX` and `CC` environment variables. -#-=- cmake_minimum_required (VERSION 3.25) -# if(NOT DEFINED DISTRIBUTION_REFROOT) -# if(DEFINED ENV{DISTRIBUTION_REFROOT}) -# set(DISTRIBUTION_REFROOT "$ENV{DISTRIBUTION_REFROOT}/" CACHE STRING "BB Dpkg root set from environment variable.") -# else() -# get_filename_component(REFROOT ${CMAKE_CURRENT_LIST_DIR}/../../../../../ REALPATH) -# set(DISTRIBUTION_REFROOT ${REFROOT}/ CACHE STRING "BB Dpkg root set from toolchain file location.") -# endif() -# endif() - -include("$ENV{DIR_SCRIPTS}/BBToolchain64.cmake") +include("$ENV{DIR_SCRIPTS}/toolchain64.cmake") if(DEFINED ENV{CC}) set(CMAKE_C_COMPILER $ENV{CC} CACHE STRING "Instrumentation C compiler" FORCE) @@ -23,6 +13,10 @@ if(DEFINED ENV{CXX}) set(CMAKE_CXX_COMPILER $ENV{CXX} CACHE STRING "Instrumentation C++ compiler" FORCE) endif() +if(DEFINED ENV{CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES}) + set(CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES $ENV{CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES}) +endif() + set(LIBCXX_BUILD_PATH "$ENV{LIBCXX_BUILD_PATH}") # Force disabling the use of Readline. This is a holdover until readline builds with -fPIC. @@ -44,8 +38,6 @@ string(CONCAT TOOLCHAIN_SHARED_FLAGS "-fdiagnostics-show-option " "-fsanitize=memory " "-fsanitize-blacklist=${MSAN_SUPPRESSION_LIST_PATH} " - "-isystem/usr/include " - "-isystem/usr/lib/llvm-18/lib/clang/18/include " ) # Apply shared flags to each language. diff --git a/.github/workflows/sanitizers/clang-libcxx-tsan.cmake b/.github/workflows/sanitizers/clang-libcxx-tsan.cmake index 9ce5d05207..22124fc7d3 100644 --- a/.github/workflows/sanitizers/clang-libcxx-tsan.cmake +++ b/.github/workflows/sanitizers/clang-libcxx-tsan.cmake @@ -1,19 +1,9 @@ # Compiler-less toolchain for building with {Clang, libc++} + ThreadSanitizer. # The actual compiler is passed via `CXX` and `CC` environment variables. -#-=- cmake_minimum_required (VERSION 3.25) -# if(NOT DEFINED DISTRIBUTION_REFROOT) -# if(DEFINED ENV{DISTRIBUTION_REFROOT}) -# set(DISTRIBUTION_REFROOT "$ENV{DISTRIBUTION_REFROOT}/" CACHE STRING "BB Dpkg root set from environment variable.") -# else() -# get_filename_component(REFROOT ${CMAKE_CURRENT_LIST_DIR}/../../../../../ REALPATH) -# set(DISTRIBUTION_REFROOT ${REFROOT}/ CACHE STRING "BB Dpkg root set from toolchain file location.") -# endif() -# endif() - -include("$ENV{DIR_SCRIPTS}/BBToolchain64.cmake") +include("$ENV{DIR_SCRIPTS}/toolchain64.cmake") if(DEFINED ENV{CC}) set(CMAKE_C_COMPILER $ENV{CC} CACHE STRING "Instrumentation C compiler" FORCE) @@ -23,6 +13,10 @@ if(DEFINED ENV{CXX}) set(CMAKE_CXX_COMPILER $ENV{CXX} CACHE STRING "Instrumentation C++ compiler" FORCE) endif() +if(DEFINED ENV{CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES}) + set(CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES $ENV{CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES}) +endif() + set(LIBCXX_BUILD_PATH "$ENV{LIBCXX_BUILD_PATH}") # Force disabling the use of Readline. This is a holdover until readline builds with -fPIC. @@ -41,8 +35,6 @@ string(CONCAT TOOLCHAIN_SHARED_FLAGS "-fno-omit-frame-pointer " "-fno-optimize-sibling-calls " "-fdiagnostics-show-option " - "-isystem/usr/include " - "-isystem/usr/lib/llvm-18/lib/clang/18/include " ) # Apply shared flags to each language. diff --git a/.github/workflows/sanitizers/clang-libcxx-ubsan.cmake b/.github/workflows/sanitizers/clang-libcxx-ubsan.cmake index 664f4b26da..bbe82574e9 100644 --- a/.github/workflows/sanitizers/clang-libcxx-ubsan.cmake +++ b/.github/workflows/sanitizers/clang-libcxx-ubsan.cmake @@ -1,19 +1,9 @@ # Compiler-less toolchain for building with {Clang, libc++} + AddressSanitizer. # The actual compiler is passed via `CXX` and `CC` environment variables. -#-=- cmake_minimum_required (VERSION 3.25) -# if(NOT DEFINED DISTRIBUTION_REFROOT) -# if(DEFINED ENV{DISTRIBUTION_REFROOT}) -# set(DISTRIBUTION_REFROOT "$ENV{DISTRIBUTION_REFROOT}/" CACHE STRING "BB Dpkg root set from environment variable.") -# else() -# get_filename_component(REFROOT ${CMAKE_CURRENT_LIST_DIR}/../../../../../ REALPATH) -# set(DISTRIBUTION_REFROOT ${REFROOT}/ CACHE STRING "BB Dpkg root set from toolchain file location.") -# endif() -# endif() - -include("$ENV{DIR_SCRIPTS}/BBToolchain64.cmake") +include("$ENV{DIR_SCRIPTS}/toolchain64.cmake") if(DEFINED ENV{CC}) set(CMAKE_C_COMPILER $ENV{CC} CACHE STRING "Instrumentation C compiler" FORCE) @@ -23,6 +13,10 @@ if(DEFINED ENV{CXX}) set(CMAKE_CXX_COMPILER $ENV{CXX} CACHE STRING "Instrumentation C++ compiler" FORCE) endif() +if(DEFINED ENV{CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES}) + set(CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES $ENV{CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES}) +endif() + set(LIBCXX_BUILD_PATH "$ENV{LIBCXX_BUILD_PATH}") # Force disabling the use of Readline. This is a holdover until readline builds with -fPIC. @@ -41,8 +35,6 @@ string(CONCAT TOOLCHAIN_SHARED_FLAGS "-fno-omit-frame-pointer " "-fno-optimize-sibling-calls " "-fdiagnostics-show-option " - "-isystem/usr/include " - "-isystem/usr/lib/llvm-18/lib/clang/18/include " ) # Apply shared flags to each language. diff --git a/.github/workflows/sanitizers/script.sh b/.github/workflows/sanitizers/script.sh deleted file mode 100644 index aedd1ea5e8..0000000000 --- a/.github/workflows/sanitizers/script.sh +++ /dev/null @@ -1,251 +0,0 @@ -#!/bin/bash - -set -eux - -# Prerequisites for LLVM installation -## NOTE: cmake : https://askubuntu.com/questions/355565/how-do-i-install-the-latest-version-of-cmake-from-the-command-line -apt update && apt install -y lsb-release wget software-properties-common gnupg git curl jq ninja-build bison libfl-dev pkg-config && rm -rf /var/lib/apt/lists/* /var/log/dpkg.log - -### NOTE: cmake : https://askubuntu.com/questions/355565/how-do-i-install-the-latest-version-of-cmake-from-the-command-line -wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null -apt-add-repository -y "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" - -# CMAKE and FOR BMQ ONLY -apt update && apt install -y cmake bison libfl-dev pkg-config - -# Install LLVM -# wget https://apt.llvm.org/llvm.sh -chmod +x llvm.sh -LLVM_VERSION=18 -./llvm.sh ${LLVM_VERSION} all - -# Create version-agnostic pointers to required LLVM binaries. -ln -sf /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang -ln -sf /usr/bin/clang++-${LLVM_VERSION} /usr/bin/clang++ -ln -sf /usr/bin/llvm-symbolizer-${LLVM_VERSION} /usr/bin/llvm-symbolizer - -# Parse sanitizers config -cfgquery() { - jq "${1}" "sanitizers.json" --raw-output -} -SANITIZER_NAME="msan" -LLVM_SANITIZER_NAME="$(cfgquery .${SANITIZER_NAME}.llvm_sanitizer_name)" -# Check if llvm specific cmake options are present for the given sanitizer -LLVM_SPECIFIC_CMAKE_OPTIONS="$(cfgquery .${SANITIZER_NAME}.llvm_specific_cmake_options)" -if [[ "$LLVM_SPECIFIC_CMAKE_OPTIONS" == null ]]; then LLVM_SPECIFIC_CMAKE_OPTIONS=""; fi - -# Set some initial constants -PARALLELISM=4 -# PARALLELISM=8 - -DIR_ROOT="/bmq" -DIR_EXTERNAL="${DIR_ROOT}/_external" -DIR_SRCS_EXT="${DIR_EXTERNAL}/srcs" -DIR_BUILD_EXT="${DIR_SRCS_EXT}/cmake.bld" - -DIR_SRC_BMQ="${DIR_SRCS_EXT}/blazingmq" -DIR_BUILD_BMQ="${DIR_SRC_BMQ}/cmake.bld/Linux" - -# :: checkoutGitRepo() subroutine ::::::::::::::::::::::::::::::::::::::::::::: -checkoutGitRepo() { - local repo=$1 - local ref=$2 - local repoDir=$3 - echo "Checking out ${repo} at ${ref}" - - local repoPath="${DIR_SRCS_EXT}/${repoDir}" - # mkdir ${repoPath} - - git clone -b ${ref} ${repo} \ - --depth 1 --single-branch --no-tags -c advice.detachedHead=false "${repoPath}" -} -github_url() { echo "https://github.com/$1.git"; } - -# :: Download external dependencies ::::::::::::::::::::::::::::::: -mkdir -p ${DIR_SRCS_EXT} - -# TODO: Download BlazingMQ -BMQ_TAG="main" -checkoutGitRepo "$(github_url bloomberg/blazingmq)" "${BMQ_TAG}" "blazingmq" - -# Download LLVM -LLVM_TAG="llvmorg-18.1.8" -checkoutGitRepo "$(github_url llvm/llvm-project)" "${LLVM_TAG}" "llvm-project" - -# Download google-benchmark -GOOGLE_BENCHMARK_TAG="v1.8.4" -checkoutGitRepo "$(github_url google/benchmark)" "${GOOGLE_BENCHMARK_TAG}" "google-benchmark" - -# Download googletest -GOOGLETEST_TAG="v1.14.0" -checkoutGitRepo "$(github_url google/googletest)" "${GOOGLETEST_TAG}" "googletest" - -# Download zlib -ZLIB_TAG="v1.3.1" -checkoutGitRepo "$(github_url madler/zlib)" "${ZLIB_TAG}" "zlib" - -# Download BlazingMQ dependencies -pushd "${DIR_EXTERNAL}" -/sanitizers/build_deps.sh only-download -popd - - -# :: Build libc++ with required instrumentation ::::::::::::::::::::::::::::::: -# -# The extent to which all dependencies to be compiled with sanitizer-support -# varies by sanitizer. MemorySanitizer is especially unforgiving: Failing to -# link against an instrumented standard library will yield many false -# positives. Concensus is that compiling libc++ with `-fsanitize=memory` is a -# significantly easier endeavor than doing the same with libstdc++ (the gcc -# standard library). -# -# We therefore opt to use libc++ here, just to ensure maximum flexibility. We -# follow build instructions from https://libcxx.llvm.org/BuildingLibcxx.html -LIBCXX_SRC_PATH="${DIR_SRCS_EXT}/llvm-project/runtimes" -LIBCXX_BUILD_PATH="${LIBCXX_SRC_PATH}/cmake.bld" - -cmake -B "${LIBCXX_BUILD_PATH}" \ - -S "${LIBCXX_SRC_PATH}" \ - -DCMAKE_BUILD_TYPE="Debug" \ - -DCMAKE_C_COMPILER="clang" \ - -DCMAKE_CXX_COMPILER="clang++" \ - -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ - -DLLVM_USE_SANITIZER="${LLVM_SANITIZER_NAME}" \ - ${LLVM_SPECIFIC_CMAKE_OPTIONS} - -cmake --build "${LIBCXX_BUILD_PATH}" -j${PARALLELISM} --target cxx cxxabi unwind generate-cxx-headers - -# Variables read by our custom CMake toolchain used to build everything else. -export LIBCXX_BUILD_PATH="$(realpath ${LIBCXX_BUILD_PATH})" -export DIR_SRC_BMQ="${DIR_SRC_BMQ}" - -# :: Build BDE + NTF :::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -TOOLCHAIN_PATH="/sanitizers/clang-libcxx-${SANITIZER_NAME}.cmake" -export CC="clang" -export CXX="clang++" -export BBS_BUILD_SYSTEM="ON" -PATH="$PATH:$(realpath ${DIR_SRCS_EXT}/bde-tools/bin)" -export PATH - -pushd ${DIR_SRCS_EXT} - -pushd "bde" -eval "$(bbs_build_env -u dbg_64_safe_cpp20 -b "${DIR_BUILD_EXT}/bde")" -bbs_build configure --toolchain "${TOOLCHAIN_PATH}" -bbs_build build -j${PARALLELISM} -bbs_build --install=/opt/bb --prefix=/ install -popd - -pushd "ntf-core" -# TODO The deprecated flag "-fcoroutines-ts" has been removed in clang -# 17.0.1, but NTF is still using it. We manually change this flag until -# the fix in issue 175307231 is resolved. -sed -i 's/fcoroutines-ts/fcoroutines/g' 'repository.cmake' - -./configure --keep \ - --prefix /opt/bb \ - --output "${DIR_BUILD_EXT}/ntf" \ - --without-warnings-as-errors \ - --without-usage-examples \ - --without-applications \ - --ufid 'dbg_64_safe_cpp20' \ - --toolchain "${TOOLCHAIN_PATH}" -make -j${PARALLELISM} -make install -popd - -# Note: Hack to circumvent faulty behavior in "nts-targets.cmake" -ln -sf "/opt/bb/include" "/opt/include" -ln -sf "/opt/bb/lib64" "/opt/lib64" - -# pushd DIR_SRCS_EXT -popd - -# :: Setup CMake options for all remaining builds ::::::::::::::::::::::::::::: -CMAKE_OPTIONS="\ - -D BUILD_BITNESS=64 \ - -D CMAKE_BUILD_TYPE=Debug \ - -D CMAKE_INSTALL_INCLUDEDIR=include \ - -D CMAKE_INSTALL_LIBDIR=lib64 \ - -D CMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_PATH}" - -# :: Build GoogleTest ::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -cmake -B "${DIR_SRCS_EXT}/googletest/cmake.bld" \ - -S "${DIR_SRCS_EXT}/googletest" ${CMAKE_OPTIONS} \ - -DCMAKE_INSTALL_PREFIX=/opt/bb -cmake --build "${DIR_SRCS_EXT}/googletest/cmake.bld" -j${PARALLELISM} -cmake --install "${DIR_SRCS_EXT}/googletest/cmake.bld" --prefix "/opt/bb" - - -# :: Build Google Benchmark ::::::::::::::::::::::::::::::::::::::::::::::::::: -cmake -B "${DIR_SRCS_EXT}/google-benchmark/cmake.bld" \ - -S "${DIR_SRCS_EXT}/google-benchmark" ${CMAKE_OPTIONS} \ - -DCMAKE_INSTALL_PREFIX=/opt/bb \ - -DBENCHMARK_DOWNLOAD_DEPENDENCIES="ON" \ - -DBENCHMARK_ENABLE_GTEST_TESTS="false" \ - -DBENCHMARK_ENABLE_TESTING="OFF" -cmake --build "${DIR_SRCS_EXT}/google-benchmark/cmake.bld" -j${PARALLELISM} -cmake --install "${DIR_SRCS_EXT}/google-benchmark/cmake.bld" --prefix "/opt/bb" - -## :: Build zlib :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -# Note: zlib has completely broken CMake install rules, so we must -# specify the install prefix *exactly* as it will be at configure -# time -# https://discourse.cmake.org/t/cmake-install-prefix-not-work/5040 -cmake -B "${DIR_SRCS_EXT}/zlib/cmake.bld" -S "${DIR_SRCS_EXT}/zlib" \ - -D CMAKE_INSTALL_PREFIX="/opt/bb" \ - ${CMAKE_OPTIONS} -# Make and install zlib. -cmake --build "${DIR_SRCS_EXT}/zlib/cmake.bld" -j${PARALLELISM} -cmake --install "${DIR_SRCS_EXT}/zlib/cmake.bld" - -# :: Build BlazingMQ :::::::::::::::::::::::::::::::::::::::::::::::::::::::::: - -PKG_CONFIG_PATH="/opt/bb/lib64/pkgconfig:/opt/bb/lib/pkgconfig:/opt/bb/share/pkgconfig:$(pkg-config --variable pc_path pkg-config)" \ -cmake -B "${DIR_BUILD_BMQ}" -S "${DIR_SRC_BMQ}" -G Ninja \ - -DBDE_BUILD_TARGET_64=ON \ - -DBDE_BUILD_TARGET_CPP17=ON \ - -DCMAKE_PREFIX_PATH="${DIR_SRCS_EXT}/bde-tools/BdeBuildSystem" \ - -DBDE_BUILD_TARGET_SAFE=1 ${CMAKE_OPTIONS} -cmake --build "${DIR_BUILD_BMQ}" -j${PARALLELISM} \ - --target all.t -v --clean-first - -# :: Create testing scripts ::::::::::::::::::::::::::::::::::::::::::::::::::: -envcfgquery() { - # Parses the '.environment' object from 'sanitizers.json', - # and outputs a string of whitespace-separated 'VAR=VAL' pairs intended to - # be used to set the environment for a command. - # e.g. 'asan' -> 'ASAN_OPTIONS="foo=bar:baz=baf" LSAN_OPTIONS="abc=fgh"' - # - echo $(cfgquery " \ - .${1}.environment | \ - to_entries | \ - map(\"\(.key)=\\\"\(.value | \ - to_entries | \ - map(\"\(.key)=\(.value)\") | \ - join(\":\"))\\\"\") | \ - join(\" \")") | - sed "s|%%SRC%%|$(realpath ${DIR_SRC_BMQ})|g" | - sed "s|%%ROOT%%|$(realpath ${DIR_ROOT})|g" -} - -mkscript() { - local cmd=${1} - local outfile=${2} - - echo '#!/usr/bin/env bash' > ${outfile} - echo "${cmd}" >> ${outfile} - chmod +x ${outfile} -} - -SANITIZER_ENV="BMQ_BUILD=$(realpath ${DIR_BUILD_BMQ}) " -SANITIZER_ENV+="BMQ_REPO=${DIR_SRC_BMQ} " -SANITIZER_ENV+="$(envcfgquery ${SANITIZER_NAME})" - -# 'run-env.sh' runs a command with environment required of the sanitizer. -mkscript "${SANITIZER_ENV} \${@}" "${DIR_BUILD_BMQ}/run-env.sh" - -# 'run-unittests.sh' runs all instrumented unit-tests. -CMD="cd $(realpath ${DIR_SRC_BMQ}) && " -CMD+="${DIR_BUILD_BMQ}/run-env.sh ctest -E mwcsys_executil.t --output-on-failure" -mkscript "${CMD}" "${DIR_BUILD_BMQ}/run-unittests.sh" diff --git a/.github/workflows/sanitizers/BBToolchain64.cmake b/.github/workflows/sanitizers/toolchain64.cmake similarity index 90% rename from .github/workflows/sanitizers/BBToolchain64.cmake rename to .github/workflows/sanitizers/toolchain64.cmake index 0636bab823..67ea1cc42c 100644 --- a/.github/workflows/sanitizers/BBToolchain64.cmake +++ b/.github/workflows/sanitizers/toolchain64.cmake @@ -1,13 +1,12 @@ -cmake_minimum_required (VERSION 3.22) - -#@PURPOSE: Configure compiler, ABI flags and directory structure. -# -#@DESCRIPTION: This is a cmake toolchain file, designed to be specified by +# Configure compiler and ABI flags. +# This is a cmake toolchain file, designed to be specified by # setting the CMAKE_TOOLCHAIN_FILE variable either as a command line parameter # by using -D or by calling "set" before the top-level CMakeLists.txt call to # project(). **This file should NOT be included directly**. The purpose # of the toolchain file is to prepare the build directory for cross-compiling. +cmake_minimum_required (VERSION 3.22) + set(IS_64BIT yes CACHE BOOL "Tool chain bitness.") # Set the compiler flags. Use the CACHE instead of the *_INIT variants @@ -25,7 +24,6 @@ set(CMAKE_C_LINK_FLAGS "-fasynchronous-unwind-tables -Wl,--enable-new-dtags -Wl, set(CMAKE_CXX_LINK_FLAGS "-fasynchronous-unwind-tables -Wl,--enable-new-dtags -Wl,--gc-sections -Wl,-uplink_timestamp___") # Set the path for looking up includes, libs and files. -# list(APPEND CMAKE_SYSTEM_PREFIX_PATH ${DISTRIBUTION_REFROOT}/opt/bb) set_property(GLOBAL PROPERTY FIND_LIBRARY_USE_LIB64_PATHS yes) # Set the installation directory for libraries. This is for use as the From 3bf72d44a2c856511232f2329e24d98d059d1349 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Fri, 12 Jul 2024 12:19:53 +0300 Subject: [PATCH 073/105] Add missed env.var Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/build_sanitizer.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 7d8b0232a4..32f33d421e 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -160,6 +160,7 @@ export DIR_SCRIPTS="${DIR_SCRIPTS}" TOOLCHAIN_PATH="${DIR_SCRIPTS}/clang-libcxx-${SANITIZER_NAME}.cmake" export CC="clang" export CXX="clang++" +export CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES="/usr/include;/usr/include/clang/${LLVM_VERSION}/include" export BBS_BUILD_SYSTEM="ON" PATH="$PATH:$(realpath ${DIR_SRCS_EXT}/bde-tools/bin)" export PATH From 26e1942d7309210aaea8ff215d23bdfcebeca062 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Fri, 12 Jul 2024 13:20:58 +0300 Subject: [PATCH 074/105] Add -DHAVE_STD_REGEX=ON for google-benchmark Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/build_sanitizer.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 32f33d421e..37e9d951f5 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -217,6 +217,7 @@ cmake -B "${DIR_SRCS_EXT}/google-benchmark/cmake.bld" \ -DCMAKE_INSTALL_PREFIX=/opt/bb \ -DBENCHMARK_DOWNLOAD_DEPENDENCIES="ON" \ -DBENCHMARK_ENABLE_GTEST_TESTS="false" \ + -DHAVE_STD_REGEX=ON \ -DBENCHMARK_ENABLE_TESTING="OFF" cmake --build "${DIR_SRCS_EXT}/google-benchmark/cmake.bld" -j${PARALLELISM} cmake --install "${DIR_SRCS_EXT}/google-benchmark/cmake.bld" --prefix "/opt/bb" From 8264a380a965983757765a01195ebc5f470fb3e7 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Fri, 12 Jul 2024 16:59:11 +0300 Subject: [PATCH 075/105] Cosmetic change Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/build_sanitizer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 37e9d951f5..cc128f8607 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -217,7 +217,7 @@ cmake -B "${DIR_SRCS_EXT}/google-benchmark/cmake.bld" \ -DCMAKE_INSTALL_PREFIX=/opt/bb \ -DBENCHMARK_DOWNLOAD_DEPENDENCIES="ON" \ -DBENCHMARK_ENABLE_GTEST_TESTS="false" \ - -DHAVE_STD_REGEX=ON \ + -DHAVE_STD_REGEX="ON" \ -DBENCHMARK_ENABLE_TESTING="OFF" cmake --build "${DIR_SRCS_EXT}/google-benchmark/cmake.bld" -j${PARALLELISM} cmake --install "${DIR_SRCS_EXT}/google-benchmark/cmake.bld" --prefix "/opt/bb" From e87dfd94326eac701f899dd62d499b2596bc0db6 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Fri, 12 Jul 2024 18:15:05 +0300 Subject: [PATCH 076/105] build_sanitizer.sh: use llvm.tar.gz Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/build_sanitizer.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index cc128f8607..5a15c0dc11 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -113,7 +113,9 @@ mkdir -p ${DIR_SRCS_EXT} # Download LLVM LLVM_TAG="llvmorg-18.1.8" -checkoutGitRepo "$(github_url llvm/llvm-project)" "${LLVM_TAG}" "llvm-project" +curl -SL "https://github.com/llvm/llvm-project/archive/refs/tags/${LLVM_TAG}.tar.gz" \ + | tar -xzC ${DIR_SRCS_EXT} +mv "${DIR_SRCS_EXT}/llvm-project-${LLVM_TAG}" "${DIR_SRCS_EXT}/llvm-project" # Download google-benchmark GOOGLE_BENCHMARK_TAG="v1.8.4" From 27a1d6ea4962a883b209ed5ffdff97005a7dd529 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Mon, 15 Jul 2024 13:27:45 +0300 Subject: [PATCH 077/105] mwcu_sharedresource.t: skip Asan for 2 tests Signed-off-by: Aleksandr Ivanov --- src/groups/mwc/mwcu/mwcu_sharedresource.t.cpp | 32 ++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/src/groups/mwc/mwcu/mwcu_sharedresource.t.cpp b/src/groups/mwc/mwcu/mwcu_sharedresource.t.cpp index 12ab445b49..72797d8e0d 100644 --- a/src/groups/mwc/mwcu/mwcu_sharedresource.t.cpp +++ b/src/groups/mwc/mwcu/mwcu_sharedresource.t.cpp @@ -220,7 +220,19 @@ static void test1_resource_creators() } // 4. exception safety - { + // TODO: Current BDE version has a memory leak which is already fixed + // (https://github.com/bloomberg/bde/commit/42950dfbcaf2c76cdabc266afacecee67da21a59) + // but not released yet. Skip address sanitizer check for now, and remove + // 'skipTest' flag when BDE version is bumped. +#if defined(__has_feature) // Clang-supported method for checking sanitizers. + static const bool skipTest = __has_feature(address_sanitizer); +#elif defined(__SANITIZE_ADDRESS__) + // GCC-supported macros for checking ASAN. + static const bool skipTest = true; +#else + static const bool skipTest = false; // Default to running the test. +#endif + if (!skipTest) { typedef mwcu::SharedResourceFactoryDeleter Deleter; @@ -504,12 +516,24 @@ static void test4_resource_reset() } // 5. exception safety - { + // TODO: Current BDE version has a memory leak which is already fixed + // (https://github.com/bloomberg/bde/commit/42950dfbcaf2c76cdabc266afacecee67da21a59) + // but not released yet. Skip address sanitizer check for now, and remove + // 'skipTest' flag when BDE version is bumped. +#if defined(__has_feature) // Clang-supported method for checking sanitizers. + static const bool skipTest = __has_feature(address_sanitizer); +#elif defined(__SANITIZE_ADDRESS__) + // GCC-supported macros for checking ASAN. + static const bool skipTest = true; +#else + static const bool skipTest = false; // Default to running the test. +#endif + if (!skipTest) { typedef mwcu::SharedResourceFactoryDeleter Deleter; - // this allocator is used to allocate the shared state and will throw - // the second time it is used + // this allocator is used to allocate the shared state and will + // throw the second time it is used bslma::TestAllocator onceAlloc; onceAlloc.setAllocationLimit(1); From c6e92a81cc52d3bd3171002026a7dc02c082a8b8 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Mon, 15 Jul 2024 13:52:53 +0300 Subject: [PATCH 078/105] mwcu_sharedresource.t: add BDE version check Signed-off-by: Aleksandr Ivanov --- src/groups/mwc/mwcu/mwcu_sharedresource.t.cpp | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/groups/mwc/mwcu/mwcu_sharedresource.t.cpp b/src/groups/mwc/mwcu/mwcu_sharedresource.t.cpp index 72797d8e0d..99675a3f89 100644 --- a/src/groups/mwc/mwcu/mwcu_sharedresource.t.cpp +++ b/src/groups/mwc/mwcu/mwcu_sharedresource.t.cpp @@ -220,10 +220,11 @@ static void test1_resource_creators() } // 4. exception safety - // TODO: Current BDE version has a memory leak which is already fixed + // Current BDE version 4.8 has a memory leak which is already fixed // (https://github.com/bloomberg/bde/commit/42950dfbcaf2c76cdabc266afacecee67da21a59) - // but not released yet. Skip address sanitizer check for now, and remove - // 'skipTest' flag when BDE version is bumped. + // but not released yet. Skip address sanitizer check for BDE version less + // than 4.9. +#if BSL_VERSION < BSL_MAKE_VERSION(4, 9) #if defined(__has_feature) // Clang-supported method for checking sanitizers. static const bool skipTest = __has_feature(address_sanitizer); #elif defined(__SANITIZE_ADDRESS__) @@ -232,6 +233,10 @@ static void test1_resource_creators() #else static const bool skipTest = false; // Default to running the test. #endif +#else + static const bool skipTest = false; // Default to running the test. +#endif + if (!skipTest) { typedef mwcu::SharedResourceFactoryDeleter Deleter; @@ -516,10 +521,11 @@ static void test4_resource_reset() } // 5. exception safety - // TODO: Current BDE version has a memory leak which is already fixed + // Current BDE version 4.8 has a memory leak which is already fixed // (https://github.com/bloomberg/bde/commit/42950dfbcaf2c76cdabc266afacecee67da21a59) - // but not released yet. Skip address sanitizer check for now, and remove - // 'skipTest' flag when BDE version is bumped. + // but not released yet. Skip address sanitizer check for BDE version less + // than 4.9. +#if BSL_VERSION < BSL_MAKE_VERSION(4, 9) #if defined(__has_feature) // Clang-supported method for checking sanitizers. static const bool skipTest = __has_feature(address_sanitizer); #elif defined(__SANITIZE_ADDRESS__) @@ -528,6 +534,10 @@ static void test4_resource_reset() #else static const bool skipTest = false; // Default to running the test. #endif +#else + static const bool skipTest = false; // Default to running the test. +#endif + if (!skipTest) { typedef mwcu::SharedResourceFactoryDeleter Deleter; From f7f3ef9fecdb0bd72d1453bb5cd2468aa0f60f00 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Mon, 15 Jul 2024 15:17:44 +0300 Subject: [PATCH 079/105] Add sanitize.yaml, rename sanitizers->sanitizer-check.yaml Signed-off-by: Aleksandr Ivanov --- .github/workflows/build.yaml | 8 +- .github/workflows/sanitize.yaml | 83 +++++++++++++++++++ .../{sanitizers.yaml => sanitizer-check.yaml} | 0 3 files changed, 87 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/sanitize.yaml rename .github/workflows/{sanitizers.yaml => sanitizer-check.yaml} (100%) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 752b9c0ee1..ce9a275341 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -61,24 +61,24 @@ jobs: build_and_run_asan: needs: build_dependencies - uses: ./.github/workflows/sanitizers.yaml + uses: ./.github/workflows/sanitizer-check.yaml with: sanitizer-name: 'asan' build_and_run_msan: needs: build_dependencies - uses: ./.github/workflows/sanitizers.yaml + uses: ./.github/workflows/sanitizer-check.yaml with: sanitizer-name: 'msan' build_and_run_tsan: needs: build_dependencies - uses: ./.github/workflows/sanitizers.yaml + uses: ./.github/workflows/sanitizer-check.yaml with: sanitizer-name: 'tsan' build_and_run_ubsan: needs: build_dependencies - uses: ./.github/workflows/sanitizers.yaml + uses: ./.github/workflows/sanitizer-check.yaml with: sanitizer-name: 'ubsan' diff --git a/.github/workflows/sanitize.yaml b/.github/workflows/sanitize.yaml new file mode 100644 index 0000000000..5f81a4860c --- /dev/null +++ b/.github/workflows/sanitize.yaml @@ -0,0 +1,83 @@ +name: Sanitize with AddressSanitizer, MemorySanitizer, ThreadSanitizer and UndefinedBehaviorSanitizer. + +on: + pull_request_review: + types: [submitted] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + build_dependencies: + name: Build deps [ubuntu] + if: github.event.review.state == 'APPROVED' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Get dependencies hash + id: get-hash + run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT + - name: Cache lookup + uses: actions/cache/restore@v4 + id: cache-lookup + with: + path: deps + key: deps-${{ steps.get-hash.outputs.deps_hash }} + lookup-only: true + - name: Set up dependencies + if: steps.cache-lookup.outputs.cache-hit != 'true' + run: | + sudo apt-get update + sudo apt-get install -qy build-essential \ + gdb \ + curl \ + python3.10 \ + python3-pip \ + cmake \ + ninja-build \ + pkg-config \ + bison \ + libfl-dev \ + libbenchmark-dev \ + libgmock-dev \ + libz-dev + - name: Fetch & Build non packaged dependencies + if: steps.cache-lookup.outputs.cache-hit != 'true' + run: | + mkdir -p deps + cd deps + ../docker/build_deps.sh + - name: Cache save + if: steps.cache-lookup.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: deps + key: deps-${{ steps.get-hash.outputs.deps_hash }} + + run_asan: + if: github.event.review.state == 'APPROVED' + needs: build_dependencies + uses: ./.github/workflows/sanitizer-check.yaml + with: + sanitizer-name: 'asan' + + run_msan: + if: github.event.review.state == 'APPROVED' + needs: build_dependencies + uses: ./.github/workflows/sanitizer-check.yaml + with: + sanitizer-name: 'msan' + + run_tsan: + needs: build_dependencies + uses: ./.github/workflows/sanitizer-check.yaml + with: + sanitizer-name: 'tsan' + + run_ubsan: + if: github.event.review.state == 'APPROVED' + needs: build_dependencies + uses: ./.github/workflows/sanitizer-check.yaml + with: + sanitizer-name: 'ubsan' diff --git a/.github/workflows/sanitizers.yaml b/.github/workflows/sanitizer-check.yaml similarity index 100% rename from .github/workflows/sanitizers.yaml rename to .github/workflows/sanitizer-check.yaml From 7fe283a7c507bdd292214719711fee54a51b763c Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Mon, 15 Jul 2024 16:16:06 +0300 Subject: [PATCH 080/105] Cleanup Signed-off-by: Aleksandr Ivanov --- .github/workflows/build.yaml | 322 ++++++++++++++++++++++++++++++-- .github/workflows/sanitize.yaml | 11 +- 2 files changed, 309 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ce9a275341..f786292613 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -4,7 +4,6 @@ on: push: branches: - main - - add_sanitizers_to_ci pull_request: branches: - main @@ -59,26 +58,311 @@ jobs: path: deps key: deps-${{ steps.get-hash.outputs.deps_hash }} - build_and_run_asan: + build_ubuntu: + name: Build [ubuntu] + runs-on: ubuntu-latest needs: build_dependencies - uses: ./.github/workflows/sanitizer-check.yaml - with: - sanitizer-name: 'asan' + steps: + - uses: actions/checkout@v4 + - name: Get dependencies hash + id: get-hash + run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT + - uses: actions/cache/restore@v4 + with: + path: deps + key: deps-${{ steps.get-hash.outputs.deps_hash }} + - name: Set up dependencies + run: | + sudo apt-get update + sudo apt-get install -qy build-essential \ + gdb \ + curl \ + python3.10 \ + python3-pip \ + cmake \ + ninja-build \ + pkg-config \ + bison \ + libfl-dev \ + libbenchmark-dev \ + libgmock-dev \ + libz-dev + - name: Install cached non packaged dependencies + working-directory: deps + run: ../docker/build_deps.sh + - name: Build BlazingMQ + env: + PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig + run: | + cmake -S . -B build/blazingmq -G Ninja \ + -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake \ + -DCMAKE_BUILD_TYPE=Debug \ + -DBDE_BUILD_TARGET_SAFE=ON \ + -DBDE_BUILD_TARGET_64=ON \ + -DBDE_BUILD_TARGET_CPP17=ON \ + -DCMAKE_PREFIX_PATH=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem \ + -DCMAKE_INSTALL_LIBDIR=lib64 + cmake --build build/blazingmq --parallel 8 --target all + - name: Clean-up build directories before caching + run: | + find . -name "*.o" -type f -delete + find . -name "*.a" -type f -delete + - uses: actions/cache@v4 + with: + path: | + build/blazingmq + deps + /opt/bb/include + key: cache-${{ github.sha }} - build_and_run_msan: + unit_tests_cxx: + name: UT [c++] + runs-on: ubuntu-latest needs: build_dependencies - uses: ./.github/workflows/sanitizer-check.yaml - with: - sanitizer-name: 'msan' + steps: + - uses: actions/checkout@v4 + - name: Get dependencies hash + id: get-hash + run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT + - uses: actions/cache/restore@v4 + with: + path: deps + key: deps-${{ steps.get-hash.outputs.deps_hash }} + - name: Set up dependencies + run: | + sudo apt-get update + sudo apt-get install -qy build-essential \ + gdb \ + curl \ + python3.10 \ + python3-pip \ + cmake \ + ninja-build \ + pkg-config \ + bison \ + libfl-dev \ + libbenchmark-dev \ + libgmock-dev \ + libz-dev + - name: Install cached non packaged dependencies + working-directory: deps + run: ../docker/build_deps.sh + - name: Build BlazingMQ UTs + env: + PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig + run: | + cmake -S . -B build/blazingmq -G Ninja \ + -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake \ + -DCMAKE_BUILD_TYPE=Debug \ + -DBDE_BUILD_TARGET_SAFE=ON \ + -DBDE_BUILD_TARGET_64=ON \ + -DBDE_BUILD_TARGET_CPP17=ON \ + -DCMAKE_PREFIX_PATH=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem \ + -DCMAKE_INSTALL_LIBDIR=lib64 + cmake --build build/blazingmq --parallel 8 --target all.t + - name: Run C++ Unit Tests + run: | + cd ${{ github.workspace }}/build/blazingmq + ctest -E mwcsys_executil.t --output-on-failure - build_and_run_tsan: - needs: build_dependencies - uses: ./.github/workflows/sanitizer-check.yaml - with: - sanitizer-name: 'tsan' + unit_tests_python: + name: UT [python] + runs-on: ubuntu-latest + needs: build_ubuntu + steps: + - uses: actions/checkout@v4 + - uses: actions/cache/@v4 + with: + path: | + build/blazingmq + deps + /opt/bb/include + key: cache-${{ github.sha }} + - name: Run Python Unit Tests + env: + PYTHONPATH: ${{ github.workspace }}/src/python + run: | + pip install -r ${{ github.workspace }}/src/python/requirements.txt + pip install --force-reinstall ruff==0.4.10 + cd ${{ github.workspace }} + src/python/bin/schemagen + src/python/bin/tweakgen + pytest src/python - build_and_run_ubsan: - needs: build_dependencies - uses: ./.github/workflows/sanitizer-check.yaml - with: - sanitizer-name: 'ubsan' + integration_tests_ubuntu: + name: IT [${{ matrix.cluster }}/${{ matrix.mode }}] + strategy: + matrix: + mode: ["legacy_mode", "fsm_mode"] + cluster: ["single", "multi"] + fail-fast: false + runs-on: ubuntu-latest + needs: build_ubuntu + steps: + - uses: actions/checkout@v4 + - uses: actions/cache/@v4 + with: + path: | + build/blazingmq + deps + /opt/bb/include + key: cache-${{ github.sha }} + - name: Run Integration Tests + run: | + pip install -r ${{ github.workspace }}/src/python/requirements.txt + ${{ github.workspace }}/src/integration-tests/run-tests "${{ matrix.mode }} and ${{ matrix.cluster }}" \ + --log-level ERROR \ + --log-file-level=info \ + --bmq-tolerate-dirty-shutdown \ + --bmq-log-dir=failure-logs \ + --bmq-log-level=INFO \ + --junitxml=integration-tests.xml \ + --tb long \ + --reruns=3 \ + -n 4 -v + + fuzz_tests_ubuntu: + name: Fuzz test [${{ matrix.request }}] + strategy: + matrix: + request: ["identity", "open_queue", "configure_queue_stream", "put", "confirm", "close_queue", "disconnect"] + fail-fast: false + runs-on: ubuntu-latest + needs: build_ubuntu + steps: + - uses: actions/checkout@v4 + - uses: actions/cache/@v4 + with: + path: | + build/blazingmq + deps + /opt/bb/include + key: cache-${{ github.sha }} + - name: Run Fuzz Test + run: | + pip install -r ${{ github.workspace }}/src/python/requirements.txt + cd src/python + python3 -m blazingmq.dev.fuzztest --broker-dir ${{ github.workspace }}/build/blazingmq/src/applications/bmqbrkr --request ${{ matrix.request }} + + build_macosx: + name: Build [macosx_${{ matrix.arch }}] + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: macos-14 + arch: arm64 + steps: + - uses: actions/checkout@v4 + - name: Set up dependencies + run: | + brew install \ + curl \ + python@3.10 \ + ninja \ + bison \ + flex \ + google-benchmark + - name: Build BlazingMQ + run: bin/build-darwin.sh + + build_and_test_bmqprometheus_plugin: + name: "Build Prometheus plugin [ubuntu]" + runs-on: ubuntu-latest + needs: build_ubuntu + steps: + - uses: actions/checkout@v4 + - uses: actions/cache/@v4 + with: + path: | + build/blazingmq + deps + /opt/bb/include + key: cache-${{ github.sha }} + - name: Set up plugins dependencies + run: | + sudo apt-get update + sudo apt-get install -qy build-essential \ + gdb \ + curl \ + python3.10 \ + cmake \ + ninja-build \ + pkg-config \ + bison \ + libfl-dev \ + libbenchmark-dev \ + libgmock-dev \ + libz-dev \ + autoconf \ + libtool + - name: Create dependency fetcher working directory + run: mkdir -p deps + - name: Fetch & Build non packaged plugins dependencies + working-directory: deps + run: ${{ github.workspace }}/src/plugins/bmqprometheus/build_prometheus_deps.sh + - name: Build plugins + env: + PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig + run: | + cmake -S . -B build/blazingmq -G Ninja \ + -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake \ + -DCMAKE_BUILD_TYPE=Debug \ + -DBDE_BUILD_TARGET_SAFE=ON \ + -DBDE_BUILD_TARGET_64=ON \ + -DBDE_BUILD_TARGET_CPP17=ON \ + -DCMAKE_PREFIX_PATH=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem \ + -DCMAKE_INSTALL_LIBDIR=lib64 \ + -DINSTALL_TARGETS=prometheus + cmake --build build/blazingmq --parallel 8 --target all + - name: Create prometheus dir + run: mkdir -p prometheus_dir + - name: Download Prometheus + run: curl -SL "https://github.com/prometheus/prometheus/releases/download/v2.45.1/prometheus-2.45.1.linux-amd64.tar.gz" | tar -xzC prometheus_dir/ + - name: Run Prometheus + run: ./prometheus_dir/prometheus-2.45.1.linux-amd64/prometheus --config.file=${{ github.workspace }}/src/plugins/bmqprometheus/tests/prometheus_localhost.yaml --web.enable-lifecycle --storage.tsdb.path=${{ github.workspace }}/prometheus_dir/data & + - name: Run Pushgateway + run: | + docker pull prom/pushgateway + docker run -d -p 9091:9091 prom/pushgateway + - name: Run BMQPrometheus plugin integration test in "pull" mode + run: ${{ github.workspace }}/src/plugins/bmqprometheus/tests/bmqprometheus_prometheusstatconsumer_test.py -p ${{ github.workspace }}/build/blazingmq -m pull --no-docker + - name: Clear Prometheus database and restart + run: | + curl -X POST "http://localhost:9090/-/quit" + rm -rf prometheus_dir/data + ./prometheus_dir/prometheus-2.45.1.linux-amd64/prometheus --config.file=${{ github.workspace }}/src/plugins/bmqprometheus/tests/prometheus_localhost.yaml --web.enable-lifecycle --storage.tsdb.path=${{ github.workspace }}/prometheus_dir/data & + - name: Run Prometheus plugin integration test in "push" mode + run: ${{ github.workspace }}/src/plugins/bmqprometheus/tests/bmqprometheus_prometheusstatconsumer_test.py -p ${{ github.workspace }}/build/blazingmq -m push --no-docker + + documentation: + name: "Documentation" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up dependencies + run: | + sudo apt-get update + sudo apt-get install -qy doxygen + - name: Build docs + run: | + doxygen Doxyfile + + shellcheck: + name: Shellcheck + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Run ShellCheck + uses: ludeeus/action-shellcheck@master + + docker_build_ubuntu: + name: "Docker [ubuntu]" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Docker Single-Node Workflow + run: docker compose -f docker/single-node/docker-compose.yaml up --build -d + - name: Docker Cluster Workflow + run: docker compose -f docker/cluster/docker-compose.yaml up --build -d + \ No newline at end of file diff --git a/.github/workflows/sanitize.yaml b/.github/workflows/sanitize.yaml index 5f81a4860c..c71460a4ae 100644 --- a/.github/workflows/sanitize.yaml +++ b/.github/workflows/sanitize.yaml @@ -11,8 +11,8 @@ concurrency: jobs: build_dependencies: name: Build deps [ubuntu] - if: github.event.review.state == 'APPROVED' runs-on: ubuntu-latest + if: github.event.review.state == 'APPROVED' steps: - uses: actions/checkout@v4 - name: Get dependencies hash @@ -58,26 +58,27 @@ jobs: run_asan: if: github.event.review.state == 'APPROVED' needs: build_dependencies - uses: ./.github/workflows/sanitizer-check.yaml + uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml with: sanitizer-name: 'asan' run_msan: if: github.event.review.state == 'APPROVED' needs: build_dependencies - uses: ./.github/workflows/sanitizer-check.yaml + uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml with: sanitizer-name: 'msan' run_tsan: + if: github.event.review.state == 'APPROVED' needs: build_dependencies - uses: ./.github/workflows/sanitizer-check.yaml + uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml with: sanitizer-name: 'tsan' run_ubsan: if: github.event.review.state == 'APPROVED' needs: build_dependencies - uses: ./.github/workflows/sanitizer-check.yaml + uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml with: sanitizer-name: 'ubsan' From 29c2557b8ca94b7d0ea88405eb22226068265deb Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Mon, 15 Jul 2024 16:32:22 +0300 Subject: [PATCH 081/105] Remove empty line Signed-off-by: Aleksandr Ivanov --- .github/workflows/build.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index f786292613..90c4e0386a 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -364,5 +364,4 @@ jobs: - name: Docker Single-Node Workflow run: docker compose -f docker/single-node/docker-compose.yaml up --build -d - name: Docker Cluster Workflow - run: docker compose -f docker/cluster/docker-compose.yaml up --build -d - \ No newline at end of file + run: docker compose -f docker/cluster/docker-compose.yaml up --build -d \ No newline at end of file From d1a111e44e77181f2d4cbbc4336c57eb5f55e84f Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Mon, 15 Jul 2024 16:35:17 +0300 Subject: [PATCH 082/105] Remove empty line Signed-off-by: Aleksandr Ivanov --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 90c4e0386a..2f83ddce62 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -364,4 +364,4 @@ jobs: - name: Docker Single-Node Workflow run: docker compose -f docker/single-node/docker-compose.yaml up --build -d - name: Docker Cluster Workflow - run: docker compose -f docker/cluster/docker-compose.yaml up --build -d \ No newline at end of file + run: docker compose -f docker/cluster/docker-compose.yaml up --build -d From 184e978392c849d83ba73b45723d9be60cb88f82 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Mon, 15 Jul 2024 16:46:48 +0300 Subject: [PATCH 083/105] Fix comment Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizer-check.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sanitizer-check.yaml b/.github/workflows/sanitizer-check.yaml index 9f26fc1b5a..9241b75183 100644 --- a/.github/workflows/sanitizer-check.yaml +++ b/.github/workflows/sanitizer-check.yaml @@ -1,4 +1,4 @@ -name: Sanitizers +name: Build and run Sanitizer on: workflow_call: From 51b43bf92a54816e7059806a2012ec60a2a9b67a Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Mon, 15 Jul 2024 16:48:14 +0300 Subject: [PATCH 084/105] Make path absolute Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizer-check.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sanitizer-check.yaml b/.github/workflows/sanitizer-check.yaml index 9241b75183..4c4d69faef 100644 --- a/.github/workflows/sanitizer-check.yaml +++ b/.github/workflows/sanitizer-check.yaml @@ -22,6 +22,6 @@ jobs: path: deps key: deps-${{ steps.get-hash.outputs.deps_hash }} - name: Build BlazingMQ and dependencies with sanitizer instrumentation - run: ./.github/workflows/sanitizers/build_sanitizer.sh ${{ inputs.sanitizer-name }} + run: ${{ github.workspace }}/.github/workflows/sanitizers/build_sanitizer.sh ${{ inputs.sanitizer-name }} - name: Run unit tests under sanitizer run: ${{ github.workspace }}/cmake.bld/Linux/run-unittests.sh From c67a1949c7b41fe654f0496a055cd5761d3969d6 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Mon, 15 Jul 2024 16:51:52 +0300 Subject: [PATCH 085/105] Fix code format Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/sanitizers.json | 88 ++++++++++---------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/.github/workflows/sanitizers/sanitizers.json b/.github/workflows/sanitizers/sanitizers.json index d50a4ca20a..63e31dbcd2 100644 --- a/.github/workflows/sanitizers/sanitizers.json +++ b/.github/workflows/sanitizers/sanitizers.json @@ -1,47 +1,47 @@ { - "asan": { - "bde_compiler_name": "asan", - "llvm_sanitizer_name": "Address", - "environment": { - "ASAN_OPTIONS": { - "external_symbolizer_path": "/usr/bin/llvm-symbolizer" - }, - "LSAN_OPTIONS": { - "log_threads": 1, - "verbosity": 0 - } + "asan": { + "bde_compiler_name": "asan", + "llvm_sanitizer_name": "Address", + "environment": { + "ASAN_OPTIONS": { + "external_symbolizer_path": "/usr/bin/llvm-symbolizer" + }, + "LSAN_OPTIONS": { + "log_threads": 1, + "verbosity": 0 + } + } + }, + "msan": { + "bde_compiler_name": "msan", + "llvm_sanitizer_name": "MemoryWithOrigins", + "llvm_specific_cmake_options": "-DLIBCXXABI_USE_LLVM_UNWINDER=OFF", + "environment": { + "MSAN_OPTIONS": { + "external_symbolizer_path": "/usr/bin/llvm-symbolizer" + } + } + }, + "tsan": { + "bde_compiler_name": "tsan", + "llvm_sanitizer_name": "Thread", + "environment": { + "TSAN_OPTIONS": { + "external_symbolizer_path": "/usr/bin/llvm-symbolizer", + "second_deadlock_stack": 1, + "suppressions": "%%SRC%%/etc/tsansup.txt" + } + } + }, + "ubsan": { + "bde_compiler_name": "ubsan", + "llvm_sanitizer_name": "Undefined", + "environment": { + "UBSAN_OPTIONS": { + "external_symbolizer_path": "/usr/bin/llvm-symbolizer", + "suppressions": "%%SRC%%/etc/ubsansup.txt", + "print_stacktrace": "1" + } + } } - }, - "msan": { - "bde_compiler_name": "msan", - "llvm_sanitizer_name": "MemoryWithOrigins", - "llvm_specific_cmake_options": "-DLIBCXXABI_USE_LLVM_UNWINDER=OFF", - "environment": { - "MSAN_OPTIONS": { - "external_symbolizer_path": "/usr/bin/llvm-symbolizer" - } - } - }, - "tsan": { - "bde_compiler_name": "tsan", - "llvm_sanitizer_name": "Thread", - "environment": { - "TSAN_OPTIONS": { - "external_symbolizer_path": "/usr/bin/llvm-symbolizer", - "second_deadlock_stack": 1, - "suppressions": "%%SRC%%/etc/tsansup.txt" - } - } - }, - "ubsan": { - "bde_compiler_name": "ubsan", - "llvm_sanitizer_name": "Undefined", - "environment": { - "UBSAN_OPTIONS": { - "external_symbolizer_path": "/usr/bin/llvm-symbolizer", - "suppressions": "%%SRC%%/etc/ubsansup.txt", - "print_stacktrace": "1" - } - } - } } From 6464a186f68ae005dd6f7fdbd7ca417498465a37 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Mon, 15 Jul 2024 17:20:50 +0300 Subject: [PATCH 086/105] Cleanup Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/clang-libcxx-asan.cmake | 1 - .github/workflows/sanitizers/clang-libcxx-msan.cmake | 1 - .github/workflows/sanitizers/clang-libcxx-tsan.cmake | 1 - .github/workflows/sanitizers/clang-libcxx-ubsan.cmake | 1 - 4 files changed, 4 deletions(-) diff --git a/.github/workflows/sanitizers/clang-libcxx-asan.cmake b/.github/workflows/sanitizers/clang-libcxx-asan.cmake index 853ab64d91..4269cac86d 100644 --- a/.github/workflows/sanitizers/clang-libcxx-asan.cmake +++ b/.github/workflows/sanitizers/clang-libcxx-asan.cmake @@ -20,7 +20,6 @@ endif() set(LIBCXX_BUILD_PATH "$ENV{LIBCXX_BUILD_PATH}") # Force disabling the use of Readline. This is a holdover until readline builds with -fPIC. -# https://bbgithub.dev.bloomberg.com/swt/readline/issues/8 set(BMQ_DISABLE_READLINE TRUE) set(TOOLCHAIN_CXX_FLAGS "${CMAKE_CXX_FLAGS_DEBUG}") diff --git a/.github/workflows/sanitizers/clang-libcxx-msan.cmake b/.github/workflows/sanitizers/clang-libcxx-msan.cmake index a8cffa36cb..63b327e251 100644 --- a/.github/workflows/sanitizers/clang-libcxx-msan.cmake +++ b/.github/workflows/sanitizers/clang-libcxx-msan.cmake @@ -20,7 +20,6 @@ endif() set(LIBCXX_BUILD_PATH "$ENV{LIBCXX_BUILD_PATH}") # Force disabling the use of Readline. This is a holdover until readline builds with -fPIC. -# https://bbgithub.dev.bloomberg.com/swt/readline/issues/8 set(BMQ_DISABLE_READLINE TRUE) set(TOOLCHAIN_CXX_FLAGS "${CMAKE_CXX_FLAGS_DEBUG}") diff --git a/.github/workflows/sanitizers/clang-libcxx-tsan.cmake b/.github/workflows/sanitizers/clang-libcxx-tsan.cmake index 22124fc7d3..1d983a43f6 100644 --- a/.github/workflows/sanitizers/clang-libcxx-tsan.cmake +++ b/.github/workflows/sanitizers/clang-libcxx-tsan.cmake @@ -20,7 +20,6 @@ endif() set(LIBCXX_BUILD_PATH "$ENV{LIBCXX_BUILD_PATH}") # Force disabling the use of Readline. This is a holdover until readline builds with -fPIC. -# https://bbgithub.dev.bloomberg.com/swt/readline/issues/8 set(BMQ_DISABLE_READLINE TRUE) set(TOOLCHAIN_CXX_FLAGS "${CMAKE_CXX_FLAGS_DEBUG}") diff --git a/.github/workflows/sanitizers/clang-libcxx-ubsan.cmake b/.github/workflows/sanitizers/clang-libcxx-ubsan.cmake index bbe82574e9..e8d1e71471 100644 --- a/.github/workflows/sanitizers/clang-libcxx-ubsan.cmake +++ b/.github/workflows/sanitizers/clang-libcxx-ubsan.cmake @@ -20,7 +20,6 @@ endif() set(LIBCXX_BUILD_PATH "$ENV{LIBCXX_BUILD_PATH}") # Force disabling the use of Readline. This is a holdover until readline builds with -fPIC. -# https://bbgithub.dev.bloomberg.com/swt/readline/issues/8 set(BMQ_DISABLE_READLINE TRUE) set(TOOLCHAIN_CXX_FLAGS "${CMAKE_CXX_FLAGS_DEBUG}") From f5cce552a6e7d55723dfbc7d3adc472e7921f3be Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Mon, 15 Jul 2024 17:39:18 +0300 Subject: [PATCH 087/105] Fix shellcheck issues Signed-off-by: Aleksandr Ivanov --- .github/workflows/build.yaml | 300 ++---------------- .../workflows/sanitizers/build_sanitizer.sh | 58 ++-- 2 files changed, 48 insertions(+), 310 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2f83ddce62..8584b2f8d5 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -4,6 +4,7 @@ on: push: branches: - main + - add_sanitizers_to_ci pull_request: branches: - main @@ -58,282 +59,29 @@ jobs: path: deps key: deps-${{ steps.get-hash.outputs.deps_hash }} - build_ubuntu: - name: Build [ubuntu] - runs-on: ubuntu-latest + run_asan: needs: build_dependencies - steps: - - uses: actions/checkout@v4 - - name: Get dependencies hash - id: get-hash - run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT - - uses: actions/cache/restore@v4 - with: - path: deps - key: deps-${{ steps.get-hash.outputs.deps_hash }} - - name: Set up dependencies - run: | - sudo apt-get update - sudo apt-get install -qy build-essential \ - gdb \ - curl \ - python3.10 \ - python3-pip \ - cmake \ - ninja-build \ - pkg-config \ - bison \ - libfl-dev \ - libbenchmark-dev \ - libgmock-dev \ - libz-dev - - name: Install cached non packaged dependencies - working-directory: deps - run: ../docker/build_deps.sh - - name: Build BlazingMQ - env: - PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig - run: | - cmake -S . -B build/blazingmq -G Ninja \ - -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake \ - -DCMAKE_BUILD_TYPE=Debug \ - -DBDE_BUILD_TARGET_SAFE=ON \ - -DBDE_BUILD_TARGET_64=ON \ - -DBDE_BUILD_TARGET_CPP17=ON \ - -DCMAKE_PREFIX_PATH=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem \ - -DCMAKE_INSTALL_LIBDIR=lib64 - cmake --build build/blazingmq --parallel 8 --target all - - name: Clean-up build directories before caching - run: | - find . -name "*.o" -type f -delete - find . -name "*.a" -type f -delete - - uses: actions/cache@v4 - with: - path: | - build/blazingmq - deps - /opt/bb/include - key: cache-${{ github.sha }} + uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml + with: + sanitizer-name: 'asan' - unit_tests_cxx: - name: UT [c++] - runs-on: ubuntu-latest + run_msan: needs: build_dependencies - steps: - - uses: actions/checkout@v4 - - name: Get dependencies hash - id: get-hash - run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT - - uses: actions/cache/restore@v4 - with: - path: deps - key: deps-${{ steps.get-hash.outputs.deps_hash }} - - name: Set up dependencies - run: | - sudo apt-get update - sudo apt-get install -qy build-essential \ - gdb \ - curl \ - python3.10 \ - python3-pip \ - cmake \ - ninja-build \ - pkg-config \ - bison \ - libfl-dev \ - libbenchmark-dev \ - libgmock-dev \ - libz-dev - - name: Install cached non packaged dependencies - working-directory: deps - run: ../docker/build_deps.sh - - name: Build BlazingMQ UTs - env: - PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig - run: | - cmake -S . -B build/blazingmq -G Ninja \ - -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake \ - -DCMAKE_BUILD_TYPE=Debug \ - -DBDE_BUILD_TARGET_SAFE=ON \ - -DBDE_BUILD_TARGET_64=ON \ - -DBDE_BUILD_TARGET_CPP17=ON \ - -DCMAKE_PREFIX_PATH=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem \ - -DCMAKE_INSTALL_LIBDIR=lib64 - cmake --build build/blazingmq --parallel 8 --target all.t - - name: Run C++ Unit Tests - run: | - cd ${{ github.workspace }}/build/blazingmq - ctest -E mwcsys_executil.t --output-on-failure - - unit_tests_python: - name: UT [python] - runs-on: ubuntu-latest - needs: build_ubuntu - steps: - - uses: actions/checkout@v4 - - uses: actions/cache/@v4 - with: - path: | - build/blazingmq - deps - /opt/bb/include - key: cache-${{ github.sha }} - - name: Run Python Unit Tests - env: - PYTHONPATH: ${{ github.workspace }}/src/python - run: | - pip install -r ${{ github.workspace }}/src/python/requirements.txt - pip install --force-reinstall ruff==0.4.10 - cd ${{ github.workspace }} - src/python/bin/schemagen - src/python/bin/tweakgen - pytest src/python - - integration_tests_ubuntu: - name: IT [${{ matrix.cluster }}/${{ matrix.mode }}] - strategy: - matrix: - mode: ["legacy_mode", "fsm_mode"] - cluster: ["single", "multi"] - fail-fast: false - runs-on: ubuntu-latest - needs: build_ubuntu - steps: - - uses: actions/checkout@v4 - - uses: actions/cache/@v4 - with: - path: | - build/blazingmq - deps - /opt/bb/include - key: cache-${{ github.sha }} - - name: Run Integration Tests - run: | - pip install -r ${{ github.workspace }}/src/python/requirements.txt - ${{ github.workspace }}/src/integration-tests/run-tests "${{ matrix.mode }} and ${{ matrix.cluster }}" \ - --log-level ERROR \ - --log-file-level=info \ - --bmq-tolerate-dirty-shutdown \ - --bmq-log-dir=failure-logs \ - --bmq-log-level=INFO \ - --junitxml=integration-tests.xml \ - --tb long \ - --reruns=3 \ - -n 4 -v - - fuzz_tests_ubuntu: - name: Fuzz test [${{ matrix.request }}] - strategy: - matrix: - request: ["identity", "open_queue", "configure_queue_stream", "put", "confirm", "close_queue", "disconnect"] - fail-fast: false - runs-on: ubuntu-latest - needs: build_ubuntu - steps: - - uses: actions/checkout@v4 - - uses: actions/cache/@v4 - with: - path: | - build/blazingmq - deps - /opt/bb/include - key: cache-${{ github.sha }} - - name: Run Fuzz Test - run: | - pip install -r ${{ github.workspace }}/src/python/requirements.txt - cd src/python - python3 -m blazingmq.dev.fuzztest --broker-dir ${{ github.workspace }}/build/blazingmq/src/applications/bmqbrkr --request ${{ matrix.request }} + uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml + with: + sanitizer-name: 'msan' - build_macosx: - name: Build [macosx_${{ matrix.arch }}] - runs-on: ${{ matrix.os }} - strategy: - matrix: - include: - - os: macos-14 - arch: arm64 - steps: - - uses: actions/checkout@v4 - - name: Set up dependencies - run: | - brew install \ - curl \ - python@3.10 \ - ninja \ - bison \ - flex \ - google-benchmark - - name: Build BlazingMQ - run: bin/build-darwin.sh + run_tsan: + needs: build_dependencies + uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml + with: + sanitizer-name: 'tsan' - build_and_test_bmqprometheus_plugin: - name: "Build Prometheus plugin [ubuntu]" - runs-on: ubuntu-latest - needs: build_ubuntu - steps: - - uses: actions/checkout@v4 - - uses: actions/cache/@v4 - with: - path: | - build/blazingmq - deps - /opt/bb/include - key: cache-${{ github.sha }} - - name: Set up plugins dependencies - run: | - sudo apt-get update - sudo apt-get install -qy build-essential \ - gdb \ - curl \ - python3.10 \ - cmake \ - ninja-build \ - pkg-config \ - bison \ - libfl-dev \ - libbenchmark-dev \ - libgmock-dev \ - libz-dev \ - autoconf \ - libtool - - name: Create dependency fetcher working directory - run: mkdir -p deps - - name: Fetch & Build non packaged plugins dependencies - working-directory: deps - run: ${{ github.workspace }}/src/plugins/bmqprometheus/build_prometheus_deps.sh - - name: Build plugins - env: - PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig - run: | - cmake -S . -B build/blazingmq -G Ninja \ - -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake \ - -DCMAKE_BUILD_TYPE=Debug \ - -DBDE_BUILD_TARGET_SAFE=ON \ - -DBDE_BUILD_TARGET_64=ON \ - -DBDE_BUILD_TARGET_CPP17=ON \ - -DCMAKE_PREFIX_PATH=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem \ - -DCMAKE_INSTALL_LIBDIR=lib64 \ - -DINSTALL_TARGETS=prometheus - cmake --build build/blazingmq --parallel 8 --target all - - name: Create prometheus dir - run: mkdir -p prometheus_dir - - name: Download Prometheus - run: curl -SL "https://github.com/prometheus/prometheus/releases/download/v2.45.1/prometheus-2.45.1.linux-amd64.tar.gz" | tar -xzC prometheus_dir/ - - name: Run Prometheus - run: ./prometheus_dir/prometheus-2.45.1.linux-amd64/prometheus --config.file=${{ github.workspace }}/src/plugins/bmqprometheus/tests/prometheus_localhost.yaml --web.enable-lifecycle --storage.tsdb.path=${{ github.workspace }}/prometheus_dir/data & - - name: Run Pushgateway - run: | - docker pull prom/pushgateway - docker run -d -p 9091:9091 prom/pushgateway - - name: Run BMQPrometheus plugin integration test in "pull" mode - run: ${{ github.workspace }}/src/plugins/bmqprometheus/tests/bmqprometheus_prometheusstatconsumer_test.py -p ${{ github.workspace }}/build/blazingmq -m pull --no-docker - - name: Clear Prometheus database and restart - run: | - curl -X POST "http://localhost:9090/-/quit" - rm -rf prometheus_dir/data - ./prometheus_dir/prometheus-2.45.1.linux-amd64/prometheus --config.file=${{ github.workspace }}/src/plugins/bmqprometheus/tests/prometheus_localhost.yaml --web.enable-lifecycle --storage.tsdb.path=${{ github.workspace }}/prometheus_dir/data & - - name: Run Prometheus plugin integration test in "push" mode - run: ${{ github.workspace }}/src/plugins/bmqprometheus/tests/bmqprometheus_prometheusstatconsumer_test.py -p ${{ github.workspace }}/build/blazingmq -m push --no-docker + run_ubsan: + needs: build_dependencies + uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml + with: + sanitizer-name: 'ubsan' documentation: name: "Documentation" @@ -355,13 +103,3 @@ jobs: - uses: actions/checkout@v4 - name: Run ShellCheck uses: ludeeus/action-shellcheck@master - - docker_build_ubuntu: - name: "Docker [ubuntu]" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Docker Single-Node Workflow - run: docker compose -f docker/single-node/docker-compose.yaml up --build -d - - name: Docker Cluster Workflow - run: docker compose -f docker/cluster/docker-compose.yaml up --build -d diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 5a15c0dc11..3c70528126 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -23,7 +23,7 @@ set -eux # :: Required arguments ::::::::::::::::::::::::::::::::::::::::::::::::::::::: -if [ -z ${1} ]; then +if [ -z "${1}" ]; then echo 'Error: Missing sanitizer name.' >&2 echo ' (Usage: build_sanitizer.sh )' >&2 exit 1 @@ -90,9 +90,9 @@ DIR_BUILD_BMQ="${DIR_SRC_BMQ}/cmake.bld/Linux" cfgquery() { jq "${1}" "${DIR_SCRIPTS}/sanitizers.json" --raw-output } -LLVM_SANITIZER_NAME="$(cfgquery .${SANITIZER_NAME}.llvm_sanitizer_name)" +LLVM_SANITIZER_NAME="$(cfgquery ."${SANITIZER_NAME}".llvm_sanitizer_name)" # Check if llvm specific cmake options are present for the given sanitizer -LLVM_SPECIFIC_CMAKE_OPTIONS="$(cfgquery .${SANITIZER_NAME}.llvm_specific_cmake_options)" +LLVM_SPECIFIC_CMAKE_OPTIONS="$(cfgquery ."${SANITIZER_NAME}".llvm_specific_cmake_options)" if [[ "$LLVM_SPECIFIC_CMAKE_OPTIONS" == null ]]; then LLVM_SPECIFIC_CMAKE_OPTIONS=""; fi checkoutGitRepo() { @@ -103,18 +103,18 @@ checkoutGitRepo() { local repoPath="${DIR_SRCS_EXT}/${repoDir}" - git clone -b ${ref} ${repo} \ + git clone -b "${ref}" "${repo}" \ --depth 1 --single-branch --no-tags -c advice.detachedHead=false "${repoPath}" } github_url() { echo "https://github.com/$1.git"; } # Download external dependencies -mkdir -p ${DIR_SRCS_EXT} +mkdir -p "${DIR_SRCS_EXT}" # Download LLVM LLVM_TAG="llvmorg-18.1.8" curl -SL "https://github.com/llvm/llvm-project/archive/refs/tags/${LLVM_TAG}.tar.gz" \ - | tar -xzC ${DIR_SRCS_EXT} + | tar -xzC "${DIR_SRCS_EXT}" mv "${DIR_SRCS_EXT}/llvm-project-${LLVM_TAG}" "${DIR_SRCS_EXT}/llvm-project" # Download google-benchmark @@ -150,12 +150,12 @@ cmake -B "${LIBCXX_BUILD_PATH}" \ -DCMAKE_CXX_COMPILER="clang++" \ -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ -DLLVM_USE_SANITIZER="${LLVM_SANITIZER_NAME}" \ - ${LLVM_SPECIFIC_CMAKE_OPTIONS} + "${LLVM_SPECIFIC_CMAKE_OPTIONS}" cmake --build "${LIBCXX_BUILD_PATH}" -j${PARALLELISM} --target cxx cxxabi unwind generate-cxx-headers # Variables read by our custom CMake toolchain used to build everything else. -export LIBCXX_BUILD_PATH="$(realpath ${LIBCXX_BUILD_PATH})" +export LIBCXX_BUILD_PATH="${LIBCXX_BUILD_PATH})" export DIR_SRC_BMQ="${DIR_SRC_BMQ}" export DIR_SCRIPTS="${DIR_SCRIPTS}" @@ -164,7 +164,7 @@ export CC="clang" export CXX="clang++" export CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES="/usr/include;/usr/include/clang/${LLVM_VERSION}/include" export BBS_BUILD_SYSTEM="ON" -PATH="$PATH:$(realpath ${DIR_SRCS_EXT}/bde-tools/bin)" +PATH="$PATH:$(realpath "${DIR_SRCS_EXT}"/bde-tools/bin)" export PATH # Build BDE + NTF @@ -207,7 +207,7 @@ CMAKE_OPTIONS="\ # Build GoogleTest cmake -B "${DIR_SRCS_EXT}/googletest/cmake.bld" \ - -S "${DIR_SRCS_EXT}/googletest" ${CMAKE_OPTIONS} \ + -S "${DIR_SRCS_EXT}/googletest" "${CMAKE_OPTIONS}" \ -DCMAKE_INSTALL_PREFIX=/opt/bb cmake --build "${DIR_SRCS_EXT}/googletest/cmake.bld" -j${PARALLELISM} cmake --install "${DIR_SRCS_EXT}/googletest/cmake.bld" --prefix "/opt/bb" @@ -215,7 +215,7 @@ cmake --install "${DIR_SRCS_EXT}/googletest/cmake.bld" --prefix "/opt/bb" # Build Google Benchmark cmake -B "${DIR_SRCS_EXT}/google-benchmark/cmake.bld" \ - -S "${DIR_SRCS_EXT}/google-benchmark" ${CMAKE_OPTIONS} \ + -S "${DIR_SRCS_EXT}/google-benchmark" "${CMAKE_OPTIONS}" \ -DCMAKE_INSTALL_PREFIX=/opt/bb \ -DBENCHMARK_DOWNLOAD_DEPENDENCIES="ON" \ -DBENCHMARK_ENABLE_GTEST_TESTS="false" \ @@ -231,7 +231,7 @@ cmake --install "${DIR_SRCS_EXT}/google-benchmark/cmake.bld" --prefix "/opt/bb" # https://discourse.cmake.org/t/cmake-install-prefix-not-work/5040 cmake -B "${DIR_SRCS_EXT}/zlib/cmake.bld" -S "${DIR_SRCS_EXT}/zlib" \ -D CMAKE_INSTALL_PREFIX="/opt/bb" \ - ${CMAKE_OPTIONS} + "${CMAKE_OPTIONS}" # Make and install zlib. cmake --build "${DIR_SRCS_EXT}/zlib/cmake.bld" -j${PARALLELISM} cmake --install "${DIR_SRCS_EXT}/zlib/cmake.bld" @@ -242,7 +242,7 @@ cmake -B "${DIR_BUILD_BMQ}" -S "${DIR_SRC_BMQ}" -G Ninja \ -DBDE_BUILD_TARGET_64=ON \ -DBDE_BUILD_TARGET_CPP17=ON \ -DCMAKE_PREFIX_PATH="${DIR_SRCS_EXT}/bde-tools/BdeBuildSystem" \ - -DBDE_BUILD_TARGET_SAFE=1 ${CMAKE_OPTIONS} + -DBDE_BUILD_TARGET_SAFE=1 "${CMAKE_OPTIONS}" cmake --build "${DIR_BUILD_BMQ}" -j${PARALLELISM} \ --target all.t -v --clean-first @@ -253,35 +253,35 @@ envcfgquery() { # be used to set the environment for a command. # e.g. 'asan' -> 'ASAN_OPTIONS="foo=bar:baz=baf" LSAN_OPTIONS="abc=fgh"' # - echo $(cfgquery " \ - .${1}.environment | \ - to_entries | \ - map(\"\(.key)=\\\"\(.value | \ - to_entries | \ - map(\"\(.key)=\(.value)\") | \ - join(\":\"))\\\"\") | \ - join(\" \")") | - sed "s|%%SRC%%|$(realpath ${DIR_SRC_BMQ})|g" | - sed "s|%%ROOT%%|$(realpath ${DIR_ROOT})|g" + "$(cfgquery " \ + .${1}.environment | \ + to_entries | \ + map(\"\(.key)=\\\"\(.value | \ + to_entries | \ + map(\"\(.key)=\(.value)\") | \ + join(\":\"))\\\"\") | \ + join(\" \")")" | + sed "s|%%SRC%%|$(realpath "${DIR_SRC_BMQ}")|g" | + sed "s|%%ROOT%%|$(realpath "${DIR_ROOT}")|g" } mkscript() { local cmd=${1} local outfile=${2} - echo '#!/usr/bin/env bash' > ${outfile} - echo "${cmd}" >> ${outfile} - chmod +x ${outfile} + echo '#!/usr/bin/env bash' > "${outfile}" + echo "${cmd}" >> "${outfile}" + chmod +x "${outfile}" } -SANITIZER_ENV="BMQ_BUILD=$(realpath ${DIR_BUILD_BMQ}) " +SANITIZER_ENV="BMQ_BUILD=$(realpath "${DIR_BUILD_BMQ}") " SANITIZER_ENV+="BMQ_REPO=${DIR_SRC_BMQ} " -SANITIZER_ENV+="$(envcfgquery ${SANITIZER_NAME})" +SANITIZER_ENV+="$(envcfgquery "${SANITIZER_NAME}")" # 'run-env.sh' runs a command with environment required of the sanitizer. mkscript "${SANITIZER_ENV} \${@}" "${DIR_BUILD_BMQ}/run-env.sh" # 'run-unittests.sh' runs all instrumented unit-tests. -CMD="cd $(realpath ${DIR_BUILD_BMQ}) && " +CMD="cd $(realpath "${DIR_BUILD_BMQ}") && " CMD+="./run-env.sh ctest -E mwcsys_executil.t --output-on-failure" mkscript "${CMD}" "${DIR_BUILD_BMQ}/run-unittests.sh" From d31b7912fae6b109882cc5bb9f6dda63d7327d40 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Mon, 15 Jul 2024 17:43:00 +0300 Subject: [PATCH 088/105] Debug CI Signed-off-by: Aleksandr Ivanov --- .github/workflows/build.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8584b2f8d5..cd6325bbbe 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -8,6 +8,7 @@ on: pull_request: branches: - main + - add_sanitizers_to_ci concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} From f86ec7012e4809f94d15450013228df488f7e037 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Mon, 15 Jul 2024 17:48:45 +0300 Subject: [PATCH 089/105] Debug CI Signed-off-by: Aleksandr Ivanov --- .github/workflows/build.yaml | 301 +++++++++++++++++++++++-- .github/workflows/sanitizer-debug.yaml | 84 +++++++ 2 files changed, 365 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/sanitizer-debug.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index cd6325bbbe..2f83ddce62 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -4,11 +4,9 @@ on: push: branches: - main - - add_sanitizers_to_ci pull_request: branches: - main - - add_sanitizers_to_ci concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -60,29 +58,282 @@ jobs: path: deps key: deps-${{ steps.get-hash.outputs.deps_hash }} - run_asan: + build_ubuntu: + name: Build [ubuntu] + runs-on: ubuntu-latest needs: build_dependencies - uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml - with: - sanitizer-name: 'asan' + steps: + - uses: actions/checkout@v4 + - name: Get dependencies hash + id: get-hash + run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT + - uses: actions/cache/restore@v4 + with: + path: deps + key: deps-${{ steps.get-hash.outputs.deps_hash }} + - name: Set up dependencies + run: | + sudo apt-get update + sudo apt-get install -qy build-essential \ + gdb \ + curl \ + python3.10 \ + python3-pip \ + cmake \ + ninja-build \ + pkg-config \ + bison \ + libfl-dev \ + libbenchmark-dev \ + libgmock-dev \ + libz-dev + - name: Install cached non packaged dependencies + working-directory: deps + run: ../docker/build_deps.sh + - name: Build BlazingMQ + env: + PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig + run: | + cmake -S . -B build/blazingmq -G Ninja \ + -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake \ + -DCMAKE_BUILD_TYPE=Debug \ + -DBDE_BUILD_TARGET_SAFE=ON \ + -DBDE_BUILD_TARGET_64=ON \ + -DBDE_BUILD_TARGET_CPP17=ON \ + -DCMAKE_PREFIX_PATH=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem \ + -DCMAKE_INSTALL_LIBDIR=lib64 + cmake --build build/blazingmq --parallel 8 --target all + - name: Clean-up build directories before caching + run: | + find . -name "*.o" -type f -delete + find . -name "*.a" -type f -delete + - uses: actions/cache@v4 + with: + path: | + build/blazingmq + deps + /opt/bb/include + key: cache-${{ github.sha }} - run_msan: + unit_tests_cxx: + name: UT [c++] + runs-on: ubuntu-latest needs: build_dependencies - uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml - with: - sanitizer-name: 'msan' + steps: + - uses: actions/checkout@v4 + - name: Get dependencies hash + id: get-hash + run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT + - uses: actions/cache/restore@v4 + with: + path: deps + key: deps-${{ steps.get-hash.outputs.deps_hash }} + - name: Set up dependencies + run: | + sudo apt-get update + sudo apt-get install -qy build-essential \ + gdb \ + curl \ + python3.10 \ + python3-pip \ + cmake \ + ninja-build \ + pkg-config \ + bison \ + libfl-dev \ + libbenchmark-dev \ + libgmock-dev \ + libz-dev + - name: Install cached non packaged dependencies + working-directory: deps + run: ../docker/build_deps.sh + - name: Build BlazingMQ UTs + env: + PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig + run: | + cmake -S . -B build/blazingmq -G Ninja \ + -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake \ + -DCMAKE_BUILD_TYPE=Debug \ + -DBDE_BUILD_TARGET_SAFE=ON \ + -DBDE_BUILD_TARGET_64=ON \ + -DBDE_BUILD_TARGET_CPP17=ON \ + -DCMAKE_PREFIX_PATH=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem \ + -DCMAKE_INSTALL_LIBDIR=lib64 + cmake --build build/blazingmq --parallel 8 --target all.t + - name: Run C++ Unit Tests + run: | + cd ${{ github.workspace }}/build/blazingmq + ctest -E mwcsys_executil.t --output-on-failure - run_tsan: - needs: build_dependencies - uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml - with: - sanitizer-name: 'tsan' + unit_tests_python: + name: UT [python] + runs-on: ubuntu-latest + needs: build_ubuntu + steps: + - uses: actions/checkout@v4 + - uses: actions/cache/@v4 + with: + path: | + build/blazingmq + deps + /opt/bb/include + key: cache-${{ github.sha }} + - name: Run Python Unit Tests + env: + PYTHONPATH: ${{ github.workspace }}/src/python + run: | + pip install -r ${{ github.workspace }}/src/python/requirements.txt + pip install --force-reinstall ruff==0.4.10 + cd ${{ github.workspace }} + src/python/bin/schemagen + src/python/bin/tweakgen + pytest src/python - run_ubsan: - needs: build_dependencies - uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml - with: - sanitizer-name: 'ubsan' + integration_tests_ubuntu: + name: IT [${{ matrix.cluster }}/${{ matrix.mode }}] + strategy: + matrix: + mode: ["legacy_mode", "fsm_mode"] + cluster: ["single", "multi"] + fail-fast: false + runs-on: ubuntu-latest + needs: build_ubuntu + steps: + - uses: actions/checkout@v4 + - uses: actions/cache/@v4 + with: + path: | + build/blazingmq + deps + /opt/bb/include + key: cache-${{ github.sha }} + - name: Run Integration Tests + run: | + pip install -r ${{ github.workspace }}/src/python/requirements.txt + ${{ github.workspace }}/src/integration-tests/run-tests "${{ matrix.mode }} and ${{ matrix.cluster }}" \ + --log-level ERROR \ + --log-file-level=info \ + --bmq-tolerate-dirty-shutdown \ + --bmq-log-dir=failure-logs \ + --bmq-log-level=INFO \ + --junitxml=integration-tests.xml \ + --tb long \ + --reruns=3 \ + -n 4 -v + + fuzz_tests_ubuntu: + name: Fuzz test [${{ matrix.request }}] + strategy: + matrix: + request: ["identity", "open_queue", "configure_queue_stream", "put", "confirm", "close_queue", "disconnect"] + fail-fast: false + runs-on: ubuntu-latest + needs: build_ubuntu + steps: + - uses: actions/checkout@v4 + - uses: actions/cache/@v4 + with: + path: | + build/blazingmq + deps + /opt/bb/include + key: cache-${{ github.sha }} + - name: Run Fuzz Test + run: | + pip install -r ${{ github.workspace }}/src/python/requirements.txt + cd src/python + python3 -m blazingmq.dev.fuzztest --broker-dir ${{ github.workspace }}/build/blazingmq/src/applications/bmqbrkr --request ${{ matrix.request }} + + build_macosx: + name: Build [macosx_${{ matrix.arch }}] + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: macos-14 + arch: arm64 + steps: + - uses: actions/checkout@v4 + - name: Set up dependencies + run: | + brew install \ + curl \ + python@3.10 \ + ninja \ + bison \ + flex \ + google-benchmark + - name: Build BlazingMQ + run: bin/build-darwin.sh + + build_and_test_bmqprometheus_plugin: + name: "Build Prometheus plugin [ubuntu]" + runs-on: ubuntu-latest + needs: build_ubuntu + steps: + - uses: actions/checkout@v4 + - uses: actions/cache/@v4 + with: + path: | + build/blazingmq + deps + /opt/bb/include + key: cache-${{ github.sha }} + - name: Set up plugins dependencies + run: | + sudo apt-get update + sudo apt-get install -qy build-essential \ + gdb \ + curl \ + python3.10 \ + cmake \ + ninja-build \ + pkg-config \ + bison \ + libfl-dev \ + libbenchmark-dev \ + libgmock-dev \ + libz-dev \ + autoconf \ + libtool + - name: Create dependency fetcher working directory + run: mkdir -p deps + - name: Fetch & Build non packaged plugins dependencies + working-directory: deps + run: ${{ github.workspace }}/src/plugins/bmqprometheus/build_prometheus_deps.sh + - name: Build plugins + env: + PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig + run: | + cmake -S . -B build/blazingmq -G Ninja \ + -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake \ + -DCMAKE_BUILD_TYPE=Debug \ + -DBDE_BUILD_TARGET_SAFE=ON \ + -DBDE_BUILD_TARGET_64=ON \ + -DBDE_BUILD_TARGET_CPP17=ON \ + -DCMAKE_PREFIX_PATH=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem \ + -DCMAKE_INSTALL_LIBDIR=lib64 \ + -DINSTALL_TARGETS=prometheus + cmake --build build/blazingmq --parallel 8 --target all + - name: Create prometheus dir + run: mkdir -p prometheus_dir + - name: Download Prometheus + run: curl -SL "https://github.com/prometheus/prometheus/releases/download/v2.45.1/prometheus-2.45.1.linux-amd64.tar.gz" | tar -xzC prometheus_dir/ + - name: Run Prometheus + run: ./prometheus_dir/prometheus-2.45.1.linux-amd64/prometheus --config.file=${{ github.workspace }}/src/plugins/bmqprometheus/tests/prometheus_localhost.yaml --web.enable-lifecycle --storage.tsdb.path=${{ github.workspace }}/prometheus_dir/data & + - name: Run Pushgateway + run: | + docker pull prom/pushgateway + docker run -d -p 9091:9091 prom/pushgateway + - name: Run BMQPrometheus plugin integration test in "pull" mode + run: ${{ github.workspace }}/src/plugins/bmqprometheus/tests/bmqprometheus_prometheusstatconsumer_test.py -p ${{ github.workspace }}/build/blazingmq -m pull --no-docker + - name: Clear Prometheus database and restart + run: | + curl -X POST "http://localhost:9090/-/quit" + rm -rf prometheus_dir/data + ./prometheus_dir/prometheus-2.45.1.linux-amd64/prometheus --config.file=${{ github.workspace }}/src/plugins/bmqprometheus/tests/prometheus_localhost.yaml --web.enable-lifecycle --storage.tsdb.path=${{ github.workspace }}/prometheus_dir/data & + - name: Run Prometheus plugin integration test in "push" mode + run: ${{ github.workspace }}/src/plugins/bmqprometheus/tests/bmqprometheus_prometheusstatconsumer_test.py -p ${{ github.workspace }}/build/blazingmq -m push --no-docker documentation: name: "Documentation" @@ -104,3 +355,13 @@ jobs: - uses: actions/checkout@v4 - name: Run ShellCheck uses: ludeeus/action-shellcheck@master + + docker_build_ubuntu: + name: "Docker [ubuntu]" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Docker Single-Node Workflow + run: docker compose -f docker/single-node/docker-compose.yaml up --build -d + - name: Docker Cluster Workflow + run: docker compose -f docker/cluster/docker-compose.yaml up --build -d diff --git a/.github/workflows/sanitizer-debug.yaml b/.github/workflows/sanitizer-debug.yaml new file mode 100644 index 0000000000..634e26c273 --- /dev/null +++ b/.github/workflows/sanitizer-debug.yaml @@ -0,0 +1,84 @@ +name: Debug + +on: + push: + branches: + - main + - add_sanitizers_to_ci + pull_request: + branches: + - main + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + build_dependencies: + name: Build deps [ubuntu] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Get dependencies hash + id: get-hash + run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT + - name: Cache lookup + uses: actions/cache/restore@v4 + id: cache-lookup + with: + path: deps + key: deps-${{ steps.get-hash.outputs.deps_hash }} + lookup-only: true + - name: Set up dependencies + if: steps.cache-lookup.outputs.cache-hit != 'true' + run: | + sudo apt-get update + sudo apt-get install -qy build-essential \ + gdb \ + curl \ + python3.10 \ + python3-pip \ + cmake \ + ninja-build \ + pkg-config \ + bison \ + libfl-dev \ + libbenchmark-dev \ + libgmock-dev \ + libz-dev + - name: Fetch & Build non packaged dependencies + if: steps.cache-lookup.outputs.cache-hit != 'true' + run: | + mkdir -p deps + cd deps + ../docker/build_deps.sh + - name: Cache save + if: steps.cache-lookup.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: deps + key: deps-${{ steps.get-hash.outputs.deps_hash }} + + run_asan: + needs: build_dependencies + uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml + with: + sanitizer-name: 'asan' + + run_msan: + needs: build_dependencies + uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml + with: + sanitizer-name: 'msan' + + run_tsan: + needs: build_dependencies + uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml + with: + sanitizer-name: 'tsan' + + run_ubsan: + needs: build_dependencies + uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml + with: + sanitizer-name: 'ubsan' From 72767a31f422f6abbc976a17620109d844f63156 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Mon, 15 Jul 2024 17:51:20 +0300 Subject: [PATCH 090/105] Debug CI Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizer-debug.yaml | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/.github/workflows/sanitizer-debug.yaml b/.github/workflows/sanitizer-debug.yaml index 634e26c273..f10acc22ed 100644 --- a/.github/workflows/sanitizer-debug.yaml +++ b/.github/workflows/sanitizer-debug.yaml @@ -1,17 +1,13 @@ -name: Debug +name: Debug sanitizers on: - push: - branches: - - main - - add_sanitizers_to_ci pull_request: - branches: - - main - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true + types: + - "opened" + - "reopened" + - "synchronize" + - "labeled" + - "unlabeled" jobs: build_dependencies: From 673e020ffa8e43e4c3d2ec37efd0e675f33c9f37 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Mon, 15 Jul 2024 17:54:50 +0300 Subject: [PATCH 091/105] Debug CI Signed-off-by: Aleksandr Ivanov --- .github/workflows/build.yaml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2f83ddce62..51d660d9bb 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -365,3 +365,27 @@ jobs: run: docker compose -f docker/single-node/docker-compose.yaml up --build -d - name: Docker Cluster Workflow run: docker compose -f docker/cluster/docker-compose.yaml up --build -d + + run_asan: + needs: build_dependencies + uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml + with: + sanitizer-name: 'asan' + + run_msan: + needs: build_dependencies + uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml + with: + sanitizer-name: 'msan' + + run_tsan: + needs: build_dependencies + uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml + with: + sanitizer-name: 'tsan' + + run_ubsan: + needs: build_dependencies + uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml + with: + sanitizer-name: 'ubsan' From 9d76bd879a74571846197a19bc2a97315f5ff07e Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Mon, 15 Jul 2024 17:58:57 +0300 Subject: [PATCH 092/105] Debug CI Signed-off-by: Aleksandr Ivanov --- .github/workflows/build.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 51d660d9bb..c2175cab6e 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -4,9 +4,11 @@ on: push: branches: - main + - add_sanitizers_to_ci pull_request: branches: - main + - add_sanitizers_to_ci concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} From ede9ccffc4181702a26c11c85e9bd7b549512fcf Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Mon, 15 Jul 2024 18:01:57 +0300 Subject: [PATCH 093/105] Debug CI Signed-off-by: Aleksandr Ivanov --- .github/workflows/build.yaml | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c2175cab6e..2f83ddce62 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -4,11 +4,9 @@ on: push: branches: - main - - add_sanitizers_to_ci pull_request: branches: - main - - add_sanitizers_to_ci concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -367,27 +365,3 @@ jobs: run: docker compose -f docker/single-node/docker-compose.yaml up --build -d - name: Docker Cluster Workflow run: docker compose -f docker/cluster/docker-compose.yaml up --build -d - - run_asan: - needs: build_dependencies - uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml - with: - sanitizer-name: 'asan' - - run_msan: - needs: build_dependencies - uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml - with: - sanitizer-name: 'msan' - - run_tsan: - needs: build_dependencies - uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml - with: - sanitizer-name: 'tsan' - - run_ubsan: - needs: build_dependencies - uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml - with: - sanitizer-name: 'ubsan' From 2cb956ee629904835c7a35d56cd04584bdbfcc50 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Mon, 15 Jul 2024 18:21:02 +0300 Subject: [PATCH 094/105] Debug CI Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitize.yaml | 8 ++++---- .github/workflows/sanitizer-debug.yaml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/sanitize.yaml b/.github/workflows/sanitize.yaml index c71460a4ae..756776b7a0 100644 --- a/.github/workflows/sanitize.yaml +++ b/.github/workflows/sanitize.yaml @@ -58,27 +58,27 @@ jobs: run_asan: if: github.event.review.state == 'APPROVED' needs: build_dependencies - uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml + uses: ./.github/workflows/sanitizer-check.yaml with: sanitizer-name: 'asan' run_msan: if: github.event.review.state == 'APPROVED' needs: build_dependencies - uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml + uses: ./.github/workflows/sanitizer-check.yaml with: sanitizer-name: 'msan' run_tsan: if: github.event.review.state == 'APPROVED' needs: build_dependencies - uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml + uses: ./.github/workflows/sanitizer-check.yaml with: sanitizer-name: 'tsan' run_ubsan: if: github.event.review.state == 'APPROVED' needs: build_dependencies - uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml + uses: ./.github/workflows/sanitizer-check.yaml with: sanitizer-name: 'ubsan' diff --git a/.github/workflows/sanitizer-debug.yaml b/.github/workflows/sanitizer-debug.yaml index f10acc22ed..558bd85bfb 100644 --- a/.github/workflows/sanitizer-debug.yaml +++ b/.github/workflows/sanitizer-debug.yaml @@ -57,24 +57,24 @@ jobs: run_asan: needs: build_dependencies - uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml + uses: ./.github/workflows/sanitizer-check.yaml with: sanitizer-name: 'asan' run_msan: needs: build_dependencies - uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml + uses: ./.github/workflows/sanitizer-check.yaml with: sanitizer-name: 'msan' run_tsan: needs: build_dependencies - uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml + uses: ./.github/workflows/sanitizer-check.yaml with: sanitizer-name: 'tsan' run_ubsan: needs: build_dependencies - uses: ${{ github.workspace }}/.github/workflows/sanitizer-check.yaml + uses: ./.github/workflows/sanitizer-check.yaml with: sanitizer-name: 'ubsan' From 4152ac73a0a4b264780040e38b64312141cf8a93 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Mon, 15 Jul 2024 18:28:05 +0300 Subject: [PATCH 095/105] Debug CI Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/build_sanitizer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 3c70528126..a685d5ce29 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -155,7 +155,7 @@ cmake -B "${LIBCXX_BUILD_PATH}" \ cmake --build "${LIBCXX_BUILD_PATH}" -j${PARALLELISM} --target cxx cxxabi unwind generate-cxx-headers # Variables read by our custom CMake toolchain used to build everything else. -export LIBCXX_BUILD_PATH="${LIBCXX_BUILD_PATH})" +export LIBCXX_BUILD_PATH="${LIBCXX_BUILD_PATH}" export DIR_SRC_BMQ="${DIR_SRC_BMQ}" export DIR_SCRIPTS="${DIR_SCRIPTS}" From 5a3f7a50723786e872b1127b861d4e561ba8f765 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 16 Jul 2024 11:42:58 +0300 Subject: [PATCH 096/105] Fix CMAKE_OPTIONS in build_sanitizer.sh Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/build_sanitizer.sh | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index a685d5ce29..f70d582aeb 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -198,24 +198,23 @@ ln -sf "/opt/bb/include" "/opt/include" ln -sf "/opt/bb/lib64" "/opt/lib64" # Setup CMake options for all remaining builds -CMAKE_OPTIONS="\ +CMAKE_OPTIONS=( \ -D BUILD_BITNESS=64 \ -D CMAKE_BUILD_TYPE=Debug \ -D CMAKE_INSTALL_INCLUDEDIR=include \ -D CMAKE_INSTALL_LIBDIR=lib64 \ - -D CMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_PATH}" + -D CMAKE_TOOLCHAIN_FILE="${TOOLCHAIN_PATH}") # Build GoogleTest cmake -B "${DIR_SRCS_EXT}/googletest/cmake.bld" \ - -S "${DIR_SRCS_EXT}/googletest" "${CMAKE_OPTIONS}" \ + -S "${DIR_SRCS_EXT}/googletest" "${CMAKE_OPTIONS[@]}" \ -DCMAKE_INSTALL_PREFIX=/opt/bb cmake --build "${DIR_SRCS_EXT}/googletest/cmake.bld" -j${PARALLELISM} cmake --install "${DIR_SRCS_EXT}/googletest/cmake.bld" --prefix "/opt/bb" - # Build Google Benchmark cmake -B "${DIR_SRCS_EXT}/google-benchmark/cmake.bld" \ - -S "${DIR_SRCS_EXT}/google-benchmark" "${CMAKE_OPTIONS}" \ + -S "${DIR_SRCS_EXT}/google-benchmark" "${CMAKE_OPTIONS[@]}" \ -DCMAKE_INSTALL_PREFIX=/opt/bb \ -DBENCHMARK_DOWNLOAD_DEPENDENCIES="ON" \ -DBENCHMARK_ENABLE_GTEST_TESTS="false" \ @@ -231,7 +230,7 @@ cmake --install "${DIR_SRCS_EXT}/google-benchmark/cmake.bld" --prefix "/opt/bb" # https://discourse.cmake.org/t/cmake-install-prefix-not-work/5040 cmake -B "${DIR_SRCS_EXT}/zlib/cmake.bld" -S "${DIR_SRCS_EXT}/zlib" \ -D CMAKE_INSTALL_PREFIX="/opt/bb" \ - "${CMAKE_OPTIONS}" + "${CMAKE_OPTIONS[@]}" # Make and install zlib. cmake --build "${DIR_SRCS_EXT}/zlib/cmake.bld" -j${PARALLELISM} cmake --install "${DIR_SRCS_EXT}/zlib/cmake.bld" @@ -242,7 +241,7 @@ cmake -B "${DIR_BUILD_BMQ}" -S "${DIR_SRC_BMQ}" -G Ninja \ -DBDE_BUILD_TARGET_64=ON \ -DBDE_BUILD_TARGET_CPP17=ON \ -DCMAKE_PREFIX_PATH="${DIR_SRCS_EXT}/bde-tools/BdeBuildSystem" \ - -DBDE_BUILD_TARGET_SAFE=1 "${CMAKE_OPTIONS}" + -DBDE_BUILD_TARGET_SAFE=1 "${CMAKE_OPTIONS[@]}" cmake --build "${DIR_BUILD_BMQ}" -j${PARALLELISM} \ --target all.t -v --clean-first From 51203f99ef1309680276c6af02e127b1e7e8d2e3 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Tue, 16 Jul 2024 16:36:19 +0300 Subject: [PATCH 097/105] Fix cfgquery() in build_sanitizer.sh Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/build_sanitizer.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index f70d582aeb..ba26741dfa 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -252,14 +252,14 @@ envcfgquery() { # be used to set the environment for a command. # e.g. 'asan' -> 'ASAN_OPTIONS="foo=bar:baz=baf" LSAN_OPTIONS="abc=fgh"' # - "$(cfgquery " \ + cfgquery " \ .${1}.environment | \ to_entries | \ map(\"\(.key)=\\\"\(.value | \ to_entries | \ map(\"\(.key)=\(.value)\") | \ join(\":\"))\\\"\") | \ - join(\" \")")" | + join(\" \")" | sed "s|%%SRC%%|$(realpath "${DIR_SRC_BMQ}")|g" | sed "s|%%ROOT%%|$(realpath "${DIR_ROOT}")|g" } From c19622c3aa83b413789adf03285e24d767fd4493 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Wed, 17 Jul 2024 12:54:57 +0300 Subject: [PATCH 098/105] tsansup.txt: suppress TestSession::arriveAtStepWithCfgs Signed-off-by: Aleksandr Ivanov --- etc/tsansup.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/tsansup.txt b/etc/tsansup.txt index ac2533e289..4056b7e308 100644 --- a/etc/tsansup.txt +++ b/etc/tsansup.txt @@ -28,6 +28,7 @@ race:BloombergLP::ball::LoggerManager::isInitialized() # It is a benign race in the test driver, but should be looked into at some # point. race:TestSession::waitForChannelClose +race:TestSession::arriveAtStepWithCfgs # Since we use mqbmock::Dispatcher in unit tests, this method does not get # enqueued correctly back to cluster-dispatcher thread, causing potential From ef64ace25e7ef7815e275ccb3b936988fe6b821f Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Wed, 17 Jul 2024 13:58:45 +0300 Subject: [PATCH 099/105] Remove debug script Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizer-debug.yaml | 80 -------------------------- 1 file changed, 80 deletions(-) delete mode 100644 .github/workflows/sanitizer-debug.yaml diff --git a/.github/workflows/sanitizer-debug.yaml b/.github/workflows/sanitizer-debug.yaml deleted file mode 100644 index 558bd85bfb..0000000000 --- a/.github/workflows/sanitizer-debug.yaml +++ /dev/null @@ -1,80 +0,0 @@ -name: Debug sanitizers - -on: - pull_request: - types: - - "opened" - - "reopened" - - "synchronize" - - "labeled" - - "unlabeled" - -jobs: - build_dependencies: - name: Build deps [ubuntu] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Get dependencies hash - id: get-hash - run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT - - name: Cache lookup - uses: actions/cache/restore@v4 - id: cache-lookup - with: - path: deps - key: deps-${{ steps.get-hash.outputs.deps_hash }} - lookup-only: true - - name: Set up dependencies - if: steps.cache-lookup.outputs.cache-hit != 'true' - run: | - sudo apt-get update - sudo apt-get install -qy build-essential \ - gdb \ - curl \ - python3.10 \ - python3-pip \ - cmake \ - ninja-build \ - pkg-config \ - bison \ - libfl-dev \ - libbenchmark-dev \ - libgmock-dev \ - libz-dev - - name: Fetch & Build non packaged dependencies - if: steps.cache-lookup.outputs.cache-hit != 'true' - run: | - mkdir -p deps - cd deps - ../docker/build_deps.sh - - name: Cache save - if: steps.cache-lookup.outputs.cache-hit != 'true' - uses: actions/cache/save@v4 - with: - path: deps - key: deps-${{ steps.get-hash.outputs.deps_hash }} - - run_asan: - needs: build_dependencies - uses: ./.github/workflows/sanitizer-check.yaml - with: - sanitizer-name: 'asan' - - run_msan: - needs: build_dependencies - uses: ./.github/workflows/sanitizer-check.yaml - with: - sanitizer-name: 'msan' - - run_tsan: - needs: build_dependencies - uses: ./.github/workflows/sanitizer-check.yaml - with: - sanitizer-name: 'tsan' - - run_ubsan: - needs: build_dependencies - uses: ./.github/workflows/sanitizer-check.yaml - with: - sanitizer-name: 'ubsan' From 46829a7a935eff9244b5aedeb7dcbcc67e461d50 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Thu, 18 Jul 2024 12:51:45 +0300 Subject: [PATCH 100/105] Optimize llvm installation: install only clang Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/build_sanitizer.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index ba26741dfa..f91eea7b12 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -63,16 +63,16 @@ wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null \ sudo apt-add-repository -y "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" sudo apt-get install -qy cmake -# Install LLVM +# Install Clang wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh -LLVM_VERSION=18 -sudo ./llvm.sh ${LLVM_VERSION} all +CLANG_VERSION=18 +sudo ./llvm.sh ${CLANG_VERSION} clang # Create version-agnostic pointers to required LLVM binaries. -sudo ln -sf /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang -sudo ln -sf /usr/bin/clang++-${LLVM_VERSION} /usr/bin/clang++ -sudo ln -sf /usr/bin/llvm-symbolizer-${LLVM_VERSION} /usr/bin/llvm-symbolizer +sudo ln -sf /usr/bin/clang-${CLANG_VERSION} /usr/bin/clang +sudo ln -sf /usr/bin/clang++-${CLANG_VERSION} /usr/bin/clang++ +sudo ln -sf /usr/bin/llvm-symbolizer-${CLANG_VERSION} /usr/bin/llvm-symbolizer # Set some initial constants PARALLELISM=8 @@ -162,7 +162,7 @@ export DIR_SCRIPTS="${DIR_SCRIPTS}" TOOLCHAIN_PATH="${DIR_SCRIPTS}/clang-libcxx-${SANITIZER_NAME}.cmake" export CC="clang" export CXX="clang++" -export CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES="/usr/include;/usr/include/clang/${LLVM_VERSION}/include" +export CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES="/usr/include;/usr/include/clang/${CLANG_VERSION}/include" export BBS_BUILD_SYSTEM="ON" PATH="$PATH:$(realpath "${DIR_SRCS_EXT}"/bde-tools/bin)" export PATH From 24d7993fbf720c321b4f47e44dc563bb7f3e8ec9 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Thu, 18 Jul 2024 13:00:46 +0300 Subject: [PATCH 101/105] Switch to stable LLVM version 17 Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/build_sanitizer.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index f91eea7b12..45d91ed288 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -63,16 +63,16 @@ wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null \ sudo apt-add-repository -y "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" sudo apt-get install -qy cmake -# Install Clang +# Install LLVM wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh -CLANG_VERSION=18 -sudo ./llvm.sh ${CLANG_VERSION} clang +LLVM_VERSION=17 +sudo ./llvm.sh ${LLVM_VERSION} # Create version-agnostic pointers to required LLVM binaries. -sudo ln -sf /usr/bin/clang-${CLANG_VERSION} /usr/bin/clang -sudo ln -sf /usr/bin/clang++-${CLANG_VERSION} /usr/bin/clang++ -sudo ln -sf /usr/bin/llvm-symbolizer-${CLANG_VERSION} /usr/bin/llvm-symbolizer +sudo ln -sf /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang +sudo ln -sf /usr/bin/clang++-${LLVM_VERSION} /usr/bin/clang++ +sudo ln -sf /usr/bin/llvm-symbolizer-${LLVM_VERSION} /usr/bin/llvm-symbolizer # Set some initial constants PARALLELISM=8 @@ -112,7 +112,7 @@ github_url() { echo "https://github.com/$1.git"; } mkdir -p "${DIR_SRCS_EXT}" # Download LLVM -LLVM_TAG="llvmorg-18.1.8" +LLVM_TAG="llvmorg-17.0.6" curl -SL "https://github.com/llvm/llvm-project/archive/refs/tags/${LLVM_TAG}.tar.gz" \ | tar -xzC "${DIR_SRCS_EXT}" mv "${DIR_SRCS_EXT}/llvm-project-${LLVM_TAG}" "${DIR_SRCS_EXT}/llvm-project" @@ -162,7 +162,7 @@ export DIR_SCRIPTS="${DIR_SCRIPTS}" TOOLCHAIN_PATH="${DIR_SCRIPTS}/clang-libcxx-${SANITIZER_NAME}.cmake" export CC="clang" export CXX="clang++" -export CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES="/usr/include;/usr/include/clang/${CLANG_VERSION}/include" +export CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES="/usr/include;/usr/include/clang/${LLVM_VERSION}/include" export BBS_BUILD_SYSTEM="ON" PATH="$PATH:$(realpath "${DIR_SRCS_EXT}"/bde-tools/bin)" export PATH From 69204472532e73f478b3094208df949ae59f11ff Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Thu, 18 Jul 2024 13:19:46 +0300 Subject: [PATCH 102/105] Restore 'all' option for LLVM install Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitizers/build_sanitizer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index 45d91ed288..c42d2606d9 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -67,7 +67,7 @@ sudo apt-get install -qy cmake wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh LLVM_VERSION=17 -sudo ./llvm.sh ${LLVM_VERSION} +sudo ./llvm.sh ${LLVM_VERSION} all # Create version-agnostic pointers to required LLVM binaries. sudo ln -sf /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang From 17435b2989ebe0a4158467e83935c04ec628aceb Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Fri, 19 Jul 2024 13:06:36 +0300 Subject: [PATCH 103/105] Use common cmake file for sanitizers, remove unneeded checks in job Signed-off-by: Aleksandr Ivanov --- .github/workflows/sanitize.yaml | 4 - .../workflows/sanitizers/build_sanitizer.sh | 3 +- .../sanitizers/clang-libcxx-sanitizer.cmake | 126 ++++++++++++++++++ 3 files changed, 128 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/sanitizers/clang-libcxx-sanitizer.cmake diff --git a/.github/workflows/sanitize.yaml b/.github/workflows/sanitize.yaml index 756776b7a0..9aa2642164 100644 --- a/.github/workflows/sanitize.yaml +++ b/.github/workflows/sanitize.yaml @@ -56,28 +56,24 @@ jobs: key: deps-${{ steps.get-hash.outputs.deps_hash }} run_asan: - if: github.event.review.state == 'APPROVED' needs: build_dependencies uses: ./.github/workflows/sanitizer-check.yaml with: sanitizer-name: 'asan' run_msan: - if: github.event.review.state == 'APPROVED' needs: build_dependencies uses: ./.github/workflows/sanitizer-check.yaml with: sanitizer-name: 'msan' run_tsan: - if: github.event.review.state == 'APPROVED' needs: build_dependencies uses: ./.github/workflows/sanitizer-check.yaml with: sanitizer-name: 'tsan' run_ubsan: - if: github.event.review.state == 'APPROVED' needs: build_dependencies uses: ./.github/workflows/sanitizer-check.yaml with: diff --git a/.github/workflows/sanitizers/build_sanitizer.sh b/.github/workflows/sanitizers/build_sanitizer.sh index c42d2606d9..435c38dac4 100755 --- a/.github/workflows/sanitizers/build_sanitizer.sh +++ b/.github/workflows/sanitizers/build_sanitizer.sh @@ -159,7 +159,8 @@ export LIBCXX_BUILD_PATH="${LIBCXX_BUILD_PATH}" export DIR_SRC_BMQ="${DIR_SRC_BMQ}" export DIR_SCRIPTS="${DIR_SCRIPTS}" -TOOLCHAIN_PATH="${DIR_SCRIPTS}/clang-libcxx-${SANITIZER_NAME}.cmake" +TOOLCHAIN_PATH="${DIR_SCRIPTS}/clang-libcxx-sanitizer.cmake" +export SANITIZER_NAME="${SANITIZER_NAME}" export CC="clang" export CXX="clang++" export CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES="/usr/include;/usr/include/clang/${LLVM_VERSION}/include" diff --git a/.github/workflows/sanitizers/clang-libcxx-sanitizer.cmake b/.github/workflows/sanitizers/clang-libcxx-sanitizer.cmake new file mode 100644 index 0000000000..de04249047 --- /dev/null +++ b/.github/workflows/sanitizers/clang-libcxx-sanitizer.cmake @@ -0,0 +1,126 @@ +# Compiler-less toolchain for building with {Clang, libc++} + Sanitizer. +# The actual compiler is passed via `CXX` and `CC` environment variables. +# Sanitizer name (asan/msan/tsan/ubsan) is passed via `SANITIZER_NAME` environment variable. + +cmake_minimum_required (VERSION 3.25) + +include("$ENV{DIR_SCRIPTS}/toolchain64.cmake") + +if(DEFINED ENV{CC}) + set(CMAKE_C_COMPILER $ENV{CC} CACHE STRING "Instrumentation C compiler" FORCE) +endif() + +if(DEFINED ENV{CXX}) + set(CMAKE_CXX_COMPILER $ENV{CXX} CACHE STRING "Instrumentation C++ compiler" FORCE) +endif() + +if(DEFINED ENV{CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES}) + set(CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES $ENV{CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES}) +endif() + +set(LIBCXX_BUILD_PATH "$ENV{LIBCXX_BUILD_PATH}") + +# Force disabling the use of Readline. This is a holdover until readline builds with -fPIC. +set(BMQ_DISABLE_READLINE TRUE) + +set(TOOLCHAIN_CXX_FLAGS "${CMAKE_CXX_FLAGS_DEBUG}") +set(TOOLCHAIN_C_FLAGS "${CMAKE_C_FLAGS_DEBUG}") + +# Build shared flags. +string(CONCAT TOOLCHAIN_SHARED_FLAGS + "${}" + "${TOOLCHAIN_SHARED_FLAGS} " + "-O0 " + "-g " + "-fno-omit-frame-pointer " + "-fdiagnostics-show-option " + ) + +# Apply shared flags to each language. +string(CONCAT TOOLCHAIN_CXX_FLAGS + "${TOOLCHAIN_CXX_FLAGS} " + "${TOOLCHAIN_SHARED_FLAGS} " + ) +string(CONCAT TOOLCHAIN_C_FLAGS + "${TOOLCHAIN_C_FLAGS} " + "${TOOLCHAIN_SHARED_FLAGS} " + ) + +# Use libc++ standard library for C++. +string(CONCAT TOOLCHAIN_CXX_FLAGS + "${TOOLCHAIN_CXX_FLAGS} " + "-stdlib=libc++ " + "-I${LIBCXX_BUILD_PATH}/include/c++/v1 " + ) + +# Suppress some warnings when building C++ code. +string(CONCAT TOOLCHAIN_CXX_FLAGS + "${TOOLCHAIN_CXX_FLAGS} " + "-Wno-c++98-compat " + "-Wno-c++98-compat-extra-semi " + "-Wno-c++98-compat-pedantic " + "-Wno-deprecated " + "-Wno-deprecated-declarations " + "-Wno-disabled-macro-expansion " + "-Wno-extra-semi-stmt " + "-Wno-inconsistent-missing-destructor-override " + "-Wno-inconsistent-missing-override " + "-Wno-old-style-cast " + "-Wno-undef " + "-Wno-zero-as-null-pointer-constant " + ) + +# Define linker flags (used for both shared-objects and executables). +string( CONCAT TOOLCHAIN_LINKER_FLAGS + "${CMAKE_LINKER_FLAGS_DEBUG}" + "-stdlib=libc++ " + "-L${LIBCXX_BUILD_PATH}/lib " + "-Wl,-rpath,${LIBCXX_BUILD_PATH}/lib " + "-lc++abi " + ) + +set( TOOLCHAIN_EXE_FLAGS "${TOOLCHAIN_LINKER_FLAGS}" ) + +macro(set_build_type type) + set(CMAKE_CXX_FLAGS_${type} + "${TOOLCHAIN_CXX_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) + set(CMAKE_C_FLAGS_${type} + "${TOOLCHAIN_C_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) + set(CMAKE_EXE_LINKER_FLAGS_${type} + "${TOOLCHAIN_EXE_FLAGS} ${TOOLCHAIN_${type}_FLAGS} -static-libsan" CACHE STRING "Default" FORCE) + set(CMAKE_SHARED_LINKER_FLAGS_${type} + "${TOOLCHAIN_LINKER_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) + set(CMAKE_MODULE_LINKER_FLAGS_${type} + "${TOOLCHAIN_LINKER_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) +endmacro() + +# Define sanitizer specific flags +set(SANITIZER_NAME $ENV{SANITIZER_NAME}) +if(SANITIZER_NAME STREQUAL "asan") + set(TOOLCHAIN_DEBUG_FLAGS "-fsanitize=address -fno-optimize-sibling-calls ") +elseif(SANITIZER_NAME STREQUAL "msan") + set(MSAN_SUPPRESSION_LIST_PATH "$ENV{DIR_SRC_BMQ}/etc/msansup.txt") + set(TOOLCHAIN_DEBUG_FLAGS "-fsanitize=memory -fsanitize-blacklist=${MSAN_SUPPRESSION_LIST_PATH} ") + # Conditionally add flags helpful for debugging MemorySanitizer issues. + if (DEBUG_MEMORY_SANITIZER) + string(CONCAT TOOLCHAIN_DEBUG_FLAGS + "${TOOLCHAIN_DEBUG_FLAGS} " + "-fsanitize-memory-track-origins=2 " + "-fno-optimize-sibling-calls " + ) + endif() +elseif(SANITIZER_NAME STREQUAL "tsan") + set(TSAN_SUPPRESSION_LIST_PATH "$ENV{DIR_SRC_BMQ}/etc/tsansup.txt") + set(THREAD_SANITIZER TRUE) + set(TOOLCHAIN_DEBUG_FLAGS "-fsanitize=thread -fno-optimize-sibling-calls ") +elseif(SANITIZER_NAME STREQUAL "ubsan") + set(TOOLCHAIN_DEBUG_FLAGS "-fsanitize=undefined -fno-optimize-sibling-calls ") +else() + message(FATAL_ERROR "Unexpected sanitizer name: ${SANITIZER_NAME}") +endif() + +# Set the final configuration variables, as understood by CMake. +set_build_type(DEBUG) + +# Disable GNU c++ extensions. +set(CMAKE_CXX_EXTENSIONS OFF) From 07801cae27530773dd79c58422a23ea0709df517 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Fri, 19 Jul 2024 13:08:49 +0300 Subject: [PATCH 104/105] Cleanup unneeded files Signed-off-by: Aleksandr Ivanov --- .../sanitizers/clang-libcxx-asan.cmake | 105 ---------------- .../sanitizers/clang-libcxx-msan.cmake | 113 ------------------ .../sanitizers/clang-libcxx-tsan.cmake | 108 ----------------- .../sanitizers/clang-libcxx-ubsan.cmake | 106 ---------------- 4 files changed, 432 deletions(-) delete mode 100644 .github/workflows/sanitizers/clang-libcxx-asan.cmake delete mode 100644 .github/workflows/sanitizers/clang-libcxx-msan.cmake delete mode 100644 .github/workflows/sanitizers/clang-libcxx-tsan.cmake delete mode 100644 .github/workflows/sanitizers/clang-libcxx-ubsan.cmake diff --git a/.github/workflows/sanitizers/clang-libcxx-asan.cmake b/.github/workflows/sanitizers/clang-libcxx-asan.cmake deleted file mode 100644 index 4269cac86d..0000000000 --- a/.github/workflows/sanitizers/clang-libcxx-asan.cmake +++ /dev/null @@ -1,105 +0,0 @@ -# Compiler-less toolchain for building with {Clang, libc++} + AddressSanitizer. -# The actual compiler is passed via `CXX` and `CC` environment variables. - -cmake_minimum_required (VERSION 3.25) - -include("$ENV{DIR_SCRIPTS}/toolchain64.cmake") - -if(DEFINED ENV{CC}) - set(CMAKE_C_COMPILER $ENV{CC} CACHE STRING "Instrumentation C compiler" FORCE) -endif() - -if(DEFINED ENV{CXX}) - set(CMAKE_CXX_COMPILER $ENV{CXX} CACHE STRING "Instrumentation C++ compiler" FORCE) -endif() - -if(DEFINED ENV{CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES}) - set(CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES $ENV{CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES}) -endif() - -set(LIBCXX_BUILD_PATH "$ENV{LIBCXX_BUILD_PATH}") - -# Force disabling the use of Readline. This is a holdover until readline builds with -fPIC. -set(BMQ_DISABLE_READLINE TRUE) - -set(TOOLCHAIN_CXX_FLAGS "${CMAKE_CXX_FLAGS_DEBUG}") -set(TOOLCHAIN_C_FLAGS "${CMAKE_C_FLAGS_DEBUG}") - -# Build shared flags. -string(CONCAT TOOLCHAIN_SHARED_FLAGS - "${}" - "${TOOLCHAIN_SHARED_FLAGS} " - "-O0 " - "-g " - "-fno-omit-frame-pointer " - "-fno-optimize-sibling-calls " - "-fdiagnostics-show-option " - ) - -# Apply shared flags to each language. -string(CONCAT TOOLCHAIN_CXX_FLAGS - "${TOOLCHAIN_CXX_FLAGS} " - "${TOOLCHAIN_SHARED_FLAGS} " - ) -string(CONCAT TOOLCHAIN_C_FLAGS - "${TOOLCHAIN_C_FLAGS} " - "${TOOLCHAIN_SHARED_FLAGS} " - ) - -# Use libc++ standard library for C++. -string(CONCAT TOOLCHAIN_CXX_FLAGS - "${TOOLCHAIN_CXX_FLAGS} " - "-stdlib=libc++ " - "-I${LIBCXX_BUILD_PATH}/include/c++/v1 " - ) - -# Suppress some warnings when building C++ code. -string(CONCAT TOOLCHAIN_CXX_FLAGS - "${TOOLCHAIN_CXX_FLAGS} " - "-Wno-c++98-compat " - "-Wno-c++98-compat-extra-semi " - "-Wno-c++98-compat-pedantic " - "-Wno-deprecated " - "-Wno-deprecated-declarations " - "-Wno-disabled-macro-expansion " - "-Wno-extra-semi-stmt " - "-Wno-inconsistent-missing-destructor-override " - "-Wno-inconsistent-missing-override " - "-Wno-old-style-cast " - "-Wno-undef " - "-Wno-zero-as-null-pointer-constant " - ) - -# Define linker flags (used for both shared-objects and executables). -string( CONCAT TOOLCHAIN_LINKER_FLAGS - "${CMAKE_LINKER_FLAGS_DEBUG}" - "-stdlib=libc++ " - "-L${LIBCXX_BUILD_PATH}/lib " - "-Wl,-rpath,${LIBCXX_BUILD_PATH}/lib " - "-lc++abi " - ) - -set( TOOLCHAIN_EXE_FLAGS "${TOOLCHAIN_LINKER_FLAGS}" ) - -macro(set_build_type type) - set(CMAKE_CXX_FLAGS_${type} - "${TOOLCHAIN_CXX_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) - set(CMAKE_C_FLAGS_${type} - "${TOOLCHAIN_C_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) - set(CMAKE_EXE_LINKER_FLAGS_${type} - "${TOOLCHAIN_EXE_FLAGS} ${TOOLCHAIN_${type}_FLAGS} -static-libsan" CACHE STRING "Default" FORCE) - set(CMAKE_SHARED_LINKER_FLAGS_${type} - "${TOOLCHAIN_LINKER_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) - set(CMAKE_MODULE_LINKER_FLAGS_${type} - "${TOOLCHAIN_LINKER_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) -endmacro() - -set( TOOLCHAIN_DEBUG_FLAGS - "-fsanitize=address " - ) - -# Set the final configuration variables, as understood by CMake. -set_build_type(DEBUG) - -# Disable GNU c++ extensions. -set(CMAKE_CXX_EXTENSIONS OFF) diff --git a/.github/workflows/sanitizers/clang-libcxx-msan.cmake b/.github/workflows/sanitizers/clang-libcxx-msan.cmake deleted file mode 100644 index 63b327e251..0000000000 --- a/.github/workflows/sanitizers/clang-libcxx-msan.cmake +++ /dev/null @@ -1,113 +0,0 @@ -# Compiler-less toolchain for building with {Clang, libc++} + MemorySanitizer. -# The actual compiler is passed via `CXX` and `CC` environment variables. - -cmake_minimum_required (VERSION 3.25) - -include("$ENV{DIR_SCRIPTS}/toolchain64.cmake") - -if(DEFINED ENV{CC}) - set(CMAKE_C_COMPILER $ENV{CC} CACHE STRING "Instrumentation C compiler" FORCE) -endif() - -if(DEFINED ENV{CXX}) - set(CMAKE_CXX_COMPILER $ENV{CXX} CACHE STRING "Instrumentation C++ compiler" FORCE) -endif() - -if(DEFINED ENV{CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES}) - set(CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES $ENV{CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES}) -endif() - -set(LIBCXX_BUILD_PATH "$ENV{LIBCXX_BUILD_PATH}") - -# Force disabling the use of Readline. This is a holdover until readline builds with -fPIC. -set(BMQ_DISABLE_READLINE TRUE) - -set(TOOLCHAIN_CXX_FLAGS "${CMAKE_CXX_FLAGS_DEBUG}") -set(TOOLCHAIN_C_FLAGS "${CMAKE_C_FLAGS_DEBUG}") - -set(MSAN_SUPPRESSION_LIST_PATH "$ENV{DIR_SRC_BMQ}/etc/msansup.txt") - -# Build shared flags. -string(CONCAT TOOLCHAIN_SHARED_FLAGS - "${}" - "${TOOLCHAIN_SHARED_FLAGS} " - "-O0 " - "-g " - "-fno-omit-frame-pointer " - "-fdiagnostics-show-option " - "-fsanitize=memory " - "-fsanitize-blacklist=${MSAN_SUPPRESSION_LIST_PATH} " - ) - -# Apply shared flags to each language. -string(CONCAT TOOLCHAIN_CXX_FLAGS - "${TOOLCHAIN_CXX_FLAGS} " - "${TOOLCHAIN_SHARED_FLAGS} " - ) -string(CONCAT TOOLCHAIN_C_FLAGS - "${TOOLCHAIN_C_FLAGS} " - "${TOOLCHAIN_SHARED_FLAGS} " - ) - -# Use libc++ standard library for C++. -string(CONCAT TOOLCHAIN_CXX_FLAGS - "${TOOLCHAIN_CXX_FLAGS} " - "-stdlib=libc++ " - "-I${LIBCXX_BUILD_PATH}/include/c++/v1 " - ) - -# Suppress some warnings when building C++ code. -string(CONCAT TOOLCHAIN_CXX_FLAGS - "${TOOLCHAIN_CXX_FLAGS} " - "-Wno-c++98-compat " - "-Wno-c++98-compat-extra-semi " - "-Wno-c++98-compat-pedantic " - "-Wno-deprecated " - "-Wno-deprecated-declarations " - "-Wno-disabled-macro-expansion " - "-Wno-extra-semi-stmt " - "-Wno-inconsistent-missing-destructor-override " - "-Wno-inconsistent-missing-override " - "-Wno-old-style-cast " - "-Wno-undef " - "-Wno-zero-as-null-pointer-constant " - ) - -# Define linker flags (used for both shared-objects and executables). -string( CONCAT TOOLCHAIN_LINKER_FLAGS - "${CMAKE_LINKER_FLAGS_DEBUG}" - "-stdlib=libc++ " - "-L${LIBCXX_BUILD_PATH}/lib " - "-Wl,-rpath,${LIBCXX_BUILD_PATH}/lib " - "-lc++abi " - ) - -set( TOOLCHAIN_EXE_FLAGS "${TOOLCHAIN_LINKER_FLAGS}" ) - -macro(set_build_type type) - set(CMAKE_CXX_FLAGS_${type} - "${TOOLCHAIN_CXX_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) - set(CMAKE_C_FLAGS_${type} - "${TOOLCHAIN_C_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) - set(CMAKE_EXE_LINKER_FLAGS_${type} - "${TOOLCHAIN_EXE_FLAGS} ${TOOLCHAIN_${type}_FLAGS} -static-libsan" CACHE STRING "Default" FORCE) - set(CMAKE_SHARED_LINKER_FLAGS_${type} - "${TOOLCHAIN_LINKER_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) - set(CMAKE_MODULE_LINKER_FLAGS_${type} - "${TOOLCHAIN_LINKER_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) -endmacro() - -# Conditionally add flags helpful for debugging MemorySanitizer issues. -if (DEBUG_MEMORY_SANITIZER) - string(CONCAT TOOLCHAIN_DEBUG_FLAGS - "${TOOLCHAIN_SHARED_FLAGS} " - "-fsanitize-memory-track-origins=2 " - "-fno-optimize-sibling-calls " - ) -endif() - -# Set the final configuration variables, as understood by CMake. -set_build_type(DEBUG) - -# Disable GNU c++ extensions. -set(CMAKE_CXX_EXTENSIONS OFF) diff --git a/.github/workflows/sanitizers/clang-libcxx-tsan.cmake b/.github/workflows/sanitizers/clang-libcxx-tsan.cmake deleted file mode 100644 index 1d983a43f6..0000000000 --- a/.github/workflows/sanitizers/clang-libcxx-tsan.cmake +++ /dev/null @@ -1,108 +0,0 @@ -# Compiler-less toolchain for building with {Clang, libc++} + ThreadSanitizer. -# The actual compiler is passed via `CXX` and `CC` environment variables. - -cmake_minimum_required (VERSION 3.25) - -include("$ENV{DIR_SCRIPTS}/toolchain64.cmake") - -if(DEFINED ENV{CC}) - set(CMAKE_C_COMPILER $ENV{CC} CACHE STRING "Instrumentation C compiler" FORCE) -endif() - -if(DEFINED ENV{CXX}) - set(CMAKE_CXX_COMPILER $ENV{CXX} CACHE STRING "Instrumentation C++ compiler" FORCE) -endif() - -if(DEFINED ENV{CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES}) - set(CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES $ENV{CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES}) -endif() - -set(LIBCXX_BUILD_PATH "$ENV{LIBCXX_BUILD_PATH}") - -# Force disabling the use of Readline. This is a holdover until readline builds with -fPIC. -set(BMQ_DISABLE_READLINE TRUE) - -set(TOOLCHAIN_CXX_FLAGS "${CMAKE_CXX_FLAGS_DEBUG}") -set(TOOLCHAIN_C_FLAGS "${CMAKE_C_FLAGS_DEBUG}") - -# Build shared flags. -string(CONCAT TOOLCHAIN_SHARED_FLAGS - "${}" - "${TOOLCHAIN_SHARED_FLAGS} " - "-O0 " - "-g " - "-fno-omit-frame-pointer " - "-fno-optimize-sibling-calls " - "-fdiagnostics-show-option " - ) - -# Apply shared flags to each language. -string(CONCAT TOOLCHAIN_CXX_FLAGS - "${TOOLCHAIN_CXX_FLAGS} " - "${TOOLCHAIN_SHARED_FLAGS} " - ) -string(CONCAT TOOLCHAIN_C_FLAGS - "${TOOLCHAIN_C_FLAGS} " - "${TOOLCHAIN_SHARED_FLAGS} " - ) - -# Use libc++ standard library for C++. -string(CONCAT TOOLCHAIN_CXX_FLAGS - "${TOOLCHAIN_CXX_FLAGS} " - "-stdlib=libc++ " - "-I${LIBCXX_BUILD_PATH}/include/c++/v1 " - ) - -# Suppress some warnings when building C++ code. -string(CONCAT TOOLCHAIN_CXX_FLAGS - "${TOOLCHAIN_CXX_FLAGS} " - "-Wno-c++98-compat " - "-Wno-c++98-compat-extra-semi " - "-Wno-c++98-compat-pedantic " - "-Wno-deprecated " - "-Wno-deprecated-declarations " - "-Wno-disabled-macro-expansion " - "-Wno-extra-semi-stmt " - "-Wno-inconsistent-missing-destructor-override " - "-Wno-inconsistent-missing-override " - "-Wno-old-style-cast " - "-Wno-undef " - "-Wno-zero-as-null-pointer-constant " - ) - -# Define linker flags (used for both shared-objects and executables). -string( CONCAT TOOLCHAIN_LINKER_FLAGS - "${CMAKE_LINKER_FLAGS_DEBUG}" - "-stdlib=libc++ " - "-L${LIBCXX_BUILD_PATH}/lib " - "-Wl,-rpath,${LIBCXX_BUILD_PATH}/lib " - "-lc++abi " - ) - -set( TOOLCHAIN_EXE_FLAGS "${TOOLCHAIN_LINKER_FLAGS}" ) - -macro(set_build_type type) - set(CMAKE_CXX_FLAGS_${type} - "${TOOLCHAIN_CXX_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) - set(CMAKE_C_FLAGS_${type} - "${TOOLCHAIN_C_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) - set(CMAKE_EXE_LINKER_FLAGS_${type} - "${TOOLCHAIN_EXE_FLAGS} ${TOOLCHAIN_${type}_FLAGS} -static-libsan" CACHE STRING "Default" FORCE) - set(CMAKE_SHARED_LINKER_FLAGS_${type} - "${TOOLCHAIN_LINKER_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) - set(CMAKE_MODULE_LINKER_FLAGS_${type} - "${TOOLCHAIN_LINKER_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) -endmacro() - -set(TSAN_SUPPRESSION_LIST_PATH "$ENV{DIR_SRC_BMQ}/etc/tsansup.txt") -set(THREAD_SANITIZER TRUE) - -set( TOOLCHAIN_DEBUG_FLAGS - "-fsanitize=thread " - ) - -# Set the final configuration variables, as understood by CMake. -set_build_type(DEBUG) - -# Disable GNU c++ extensions. -set(CMAKE_CXX_EXTENSIONS OFF) diff --git a/.github/workflows/sanitizers/clang-libcxx-ubsan.cmake b/.github/workflows/sanitizers/clang-libcxx-ubsan.cmake deleted file mode 100644 index e8d1e71471..0000000000 --- a/.github/workflows/sanitizers/clang-libcxx-ubsan.cmake +++ /dev/null @@ -1,106 +0,0 @@ -# Compiler-less toolchain for building with {Clang, libc++} + AddressSanitizer. -# The actual compiler is passed via `CXX` and `CC` environment variables. - -cmake_minimum_required (VERSION 3.25) - -include("$ENV{DIR_SCRIPTS}/toolchain64.cmake") - -if(DEFINED ENV{CC}) - set(CMAKE_C_COMPILER $ENV{CC} CACHE STRING "Instrumentation C compiler" FORCE) -endif() - -if(DEFINED ENV{CXX}) - set(CMAKE_CXX_COMPILER $ENV{CXX} CACHE STRING "Instrumentation C++ compiler" FORCE) -endif() - -if(DEFINED ENV{CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES}) - set(CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES $ENV{CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES}) -endif() - -set(LIBCXX_BUILD_PATH "$ENV{LIBCXX_BUILD_PATH}") - -# Force disabling the use of Readline. This is a holdover until readline builds with -fPIC. -set(BMQ_DISABLE_READLINE TRUE) - -set(TOOLCHAIN_CXX_FLAGS "${CMAKE_CXX_FLAGS_DEBUG}") -set(TOOLCHAIN_C_FLAGS "${CMAKE_C_FLAGS_DEBUG}") - -# Build shared flags. -string(CONCAT TOOLCHAIN_SHARED_FLAGS - "${}" - "${TOOLCHAIN_SHARED_FLAGS} " - "-O0 " - "-g " - "-fno-omit-frame-pointer " - "-fno-optimize-sibling-calls " - "-fdiagnostics-show-option " - ) - -# Apply shared flags to each language. -string(CONCAT TOOLCHAIN_CXX_FLAGS - "${TOOLCHAIN_CXX_FLAGS} " - "${TOOLCHAIN_SHARED_FLAGS} " - ) -string(CONCAT TOOLCHAIN_C_FLAGS - "${TOOLCHAIN_C_FLAGS} " - "${TOOLCHAIN_SHARED_FLAGS} " - ) - -# Use libc++ standard library for C++. -string(CONCAT TOOLCHAIN_CXX_FLAGS - "${TOOLCHAIN_CXX_FLAGS} " - "-stdlib=libc++ " - "-I${LIBCXX_BUILD_PATH}/include/c++/v1 " - ) - -# Suppress some warnings when building C++ code. -string(CONCAT TOOLCHAIN_CXX_FLAGS - "${TOOLCHAIN_CXX_FLAGS} " - "-Wno-c++98-compat " - "-Wno-c++98-compat-extra-semi " - "-Wno-c++98-compat-pedantic " - "-Wno-deprecated " - "-Wno-deprecated-declarations " - "-Wno-disabled-macro-expansion " - "-Wno-extra-semi-stmt " - "-Wno-inconsistent-missing-destructor-override " - "-Wno-inconsistent-missing-override " - "-Wno-old-style-cast " - "-Wno-undef " - "-Wno-zero-as-null-pointer-constant " - ) - -# Define linker flags (used for both shared-objects and executables). -string( CONCAT TOOLCHAIN_LINKER_FLAGS - "${CMAKE_LINKER_FLAGS_DEBUG}" - "-stdlib=libc++ " - "-L${LIBCXX_BUILD_PATH}/lib " - "-Wl,-rpath,${LIBCXX_BUILD_PATH}/lib " - "-lc++abi " - ) - -set( TOOLCHAIN_EXE_FLAGS "${TOOLCHAIN_LINKER_FLAGS}" ) - -macro(set_build_type type) - set(CMAKE_CXX_FLAGS_${type} - "${TOOLCHAIN_CXX_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) - set(CMAKE_C_FLAGS_${type} - "${TOOLCHAIN_C_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) - set(CMAKE_EXE_LINKER_FLAGS_${type} - "${TOOLCHAIN_EXE_FLAGS} ${TOOLCHAIN_${type}_FLAGS} -static-libsan" CACHE STRING "Default" FORCE) - set(CMAKE_SHARED_LINKER_FLAGS_${type} - "${TOOLCHAIN_LINKER_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) - set(CMAKE_MODULE_LINKER_FLAGS_${type} - "${TOOLCHAIN_LINKER_FLAGS} ${TOOLCHAIN_${type}_FLAGS}" CACHE STRING "Default" FORCE) -endmacro() - -# Add MemorySanitizer-related flags. -set( TOOLCHAIN_DEBUG_FLAGS - "-fsanitize=undefined " - ) - -# Set the final configuration variables, as understood by CMake. -set_build_type(DEBUG) - -# Disable GNU c++ extensions. -set(CMAKE_CXX_EXTENSIONS OFF) From 13dfce41a4a5c69483592f5579397cf9a5dabc44 Mon Sep 17 00:00:00 2001 From: Aleksandr Ivanov Date: Mon, 22 Jul 2024 12:10:47 +0300 Subject: [PATCH 105/105] Cleanup unneeded flags in cmake files Signed-off-by: Aleksandr Ivanov --- .../sanitizers/clang-libcxx-sanitizer.cmake | 13 ++++--------- .github/workflows/sanitizers/toolchain64.cmake | 2 -- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/.github/workflows/sanitizers/clang-libcxx-sanitizer.cmake b/.github/workflows/sanitizers/clang-libcxx-sanitizer.cmake index de04249047..40cf97580c 100644 --- a/.github/workflows/sanitizers/clang-libcxx-sanitizer.cmake +++ b/.github/workflows/sanitizers/clang-libcxx-sanitizer.cmake @@ -20,9 +20,6 @@ endif() set(LIBCXX_BUILD_PATH "$ENV{LIBCXX_BUILD_PATH}") -# Force disabling the use of Readline. This is a holdover until readline builds with -fPIC. -set(BMQ_DISABLE_READLINE TRUE) - set(TOOLCHAIN_CXX_FLAGS "${CMAKE_CXX_FLAGS_DEBUG}") set(TOOLCHAIN_C_FLAGS "${CMAKE_C_FLAGS_DEBUG}") @@ -33,6 +30,7 @@ string(CONCAT TOOLCHAIN_SHARED_FLAGS "-O0 " "-g " "-fno-omit-frame-pointer " + "-fno-optimize-sibling-calls " "-fdiagnostics-show-option " ) @@ -97,7 +95,7 @@ endmacro() # Define sanitizer specific flags set(SANITIZER_NAME $ENV{SANITIZER_NAME}) if(SANITIZER_NAME STREQUAL "asan") - set(TOOLCHAIN_DEBUG_FLAGS "-fsanitize=address -fno-optimize-sibling-calls ") + set(TOOLCHAIN_DEBUG_FLAGS "-fsanitize=address ") elseif(SANITIZER_NAME STREQUAL "msan") set(MSAN_SUPPRESSION_LIST_PATH "$ENV{DIR_SRC_BMQ}/etc/msansup.txt") set(TOOLCHAIN_DEBUG_FLAGS "-fsanitize=memory -fsanitize-blacklist=${MSAN_SUPPRESSION_LIST_PATH} ") @@ -106,15 +104,12 @@ elseif(SANITIZER_NAME STREQUAL "msan") string(CONCAT TOOLCHAIN_DEBUG_FLAGS "${TOOLCHAIN_DEBUG_FLAGS} " "-fsanitize-memory-track-origins=2 " - "-fno-optimize-sibling-calls " ) endif() elseif(SANITIZER_NAME STREQUAL "tsan") - set(TSAN_SUPPRESSION_LIST_PATH "$ENV{DIR_SRC_BMQ}/etc/tsansup.txt") - set(THREAD_SANITIZER TRUE) - set(TOOLCHAIN_DEBUG_FLAGS "-fsanitize=thread -fno-optimize-sibling-calls ") + set(TOOLCHAIN_DEBUG_FLAGS "-fsanitize=thread ") elseif(SANITIZER_NAME STREQUAL "ubsan") - set(TOOLCHAIN_DEBUG_FLAGS "-fsanitize=undefined -fno-optimize-sibling-calls ") + set(TOOLCHAIN_DEBUG_FLAGS "-fsanitize=undefined ") else() message(FATAL_ERROR "Unexpected sanitizer name: ${SANITIZER_NAME}") endif() diff --git a/.github/workflows/sanitizers/toolchain64.cmake b/.github/workflows/sanitizers/toolchain64.cmake index 67ea1cc42c..45ddcd12a6 100644 --- a/.github/workflows/sanitizers/toolchain64.cmake +++ b/.github/workflows/sanitizers/toolchain64.cmake @@ -12,11 +12,9 @@ set(IS_64BIT yes CACHE BOOL "Tool chain bitness.") # Set the compiler flags. Use the CACHE instead of the *_INIT variants # which are modified in the in the compiler initialization modules. set(CMAKE_C_FLAGS "-march=westmere -m64 -fno-strict-aliasing" CACHE STRING "ABI C flags.") -set(CMAKE_C_FLAGS_RELWITHDEBINFO "-g -O2 -fno-omit-frame-pointer" CACHE STRING "ABI C flags.") set(CMAKE_CXX_STANDARD 20 CACHE STRING "C++ standard.") set(CMAKE_CXX_STANDARD_REQUIRED 1 CACHE BOOL "Standard version is required.") set(CMAKE_CXX_FLAGS "-D_GLIBCXX_USE_CXX11_ABI=0 -march=westmere -m64 -fno-strict-aliasing" CACHE STRING "ABI C++ flags.") -set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-g -O2 -fno-omit-frame-pointer" CACHE STRING "ABI C++ flags.") # Compile variable overrides. set(CMAKE_USER_MAKE_RULES_OVERRIDE ${CMAKE_CURRENT_LIST_FILE}) # Sets this file as the override for the builtin compiler variables and rules