Merge branch 'master' into test/cli-table

.github/workflows/build_template.yml

@@ -23,6 +23,10 @@ on:
         required: false
         type: number
         default: 0
+      UNITY_BUILD:
+        required: false
+        type: number
+        default: 1
       cmake_command:
         required: false
         type: string
@@ -63,7 +67,7 @@ jobs:
         shell: bash
     timeout-minutes: 30
     container:
-      image: manticoresearch/external_toolchain:clang16_cmake3263
+      image: manticoresearch/external_toolchain:vcpkg331_20250114
       env:
         CACHEB: "../cache"
         DIAGNOSTIC: 1
@@ -76,6 +80,7 @@ jobs:
         CTEST_CMAKE_GENERATOR: ${{ inputs.CTEST_CMAKE_GENERATOR }}
         CTEST_CONFIGURATION_TYPE: ${{ inputs.CTEST_CONFIGURATION_TYPE }}
         WITH_COVERAGE: ${{ inputs.WITH_COVERAGE }}
+        UNITY_BUILD: ${{ inputs.UNITY_BUILD }}
         SYSROOT_URL: https://repo.manticoresearch.com/repository/sysroots
         HOMEBREW_PREFIX: ${{ inputs.HOMEBREW_PREFIX }}
         PACK_GALERA: ${{ inputs.PACK_GALERA }}
@@ -106,7 +111,8 @@ jobs:
           key: ${{ inputs.cache_key }}
 
       - name: 🚀 Build
-        run: ${{ inputs.cmake_command }}
+        run: |
+          ${{ inputs.cmake_command }}
 
       - name: Upload build artifacts
         if: success()

.github/workflows/pack_publish.yml

@@ -87,6 +87,7 @@ jobs:
         mkdir build
         cd build
         cmake -DPACK=1 ..
+        export CMAKE_TOOLCHAIN_FILE=$(pwd)/dist/build_dockers/cross/linux.cmake
         cmake --build . --target package
       cache_key: pack_${{ matrix.DISTR }}_${{ matrix.arch }}
       artifact_list: "build/manticore*deb"
@@ -110,6 +111,7 @@ jobs:
         mkdir build
         cd build
         cmake -DPACK=1 ..
+        export CMAKE_TOOLCHAIN_FILE=$(pwd)/dist/build_dockers/cross/linux.cmake
         cmake --build . --target package
       cache_key: pack_${{ matrix.DISTR }}_${{ matrix.arch }}
       artifact_list: "build/manticore*rpm"
@@ -131,6 +133,7 @@ jobs:
         mkdir build
         cd build
         cmake -DPACK=1 ..
+        export CMAKE_TOOLCHAIN_FILE=$(pwd)/dist/build_dockers/cross/osx.cmake
         cmake --build . --target package
       cache_key: pack_${{ matrix.DISTR }}_${{ matrix.arch }}
       artifact_list: "build/manticore*tar.gz"
@@ -142,12 +145,13 @@ jobs:
     with:
       DISTR: windows
       arch: x64
-      sysroot_url_key: roots_mysql83_24oct12_1
-      boost_url_key: boost80_zstd_zlib
+      sysroot_url_key: roots_mysql83_jan17
+      boost_url_key: boost_80
       cmake_command: |
         mkdir build
         cd build
         cmake -DPACK=1 ..
+        export CMAKE_TOOLCHAIN_FILE=$(pwd)/dist/build_dockers/cross/windows.cmake
         cmake --build . --target package
       cache_key: pack_windows_x64
       artifact_list: "build/manticore*exe build/manticore*zip"

.github/workflows/test.yml

@@ -99,6 +99,9 @@ jobs:
     with:
       CTEST_CONFIGURATION_TYPE: "Debug"
       artifact_name: debug_build
+      cmake_command: |
+        export CMAKE_TOOLCHAIN_FILE=$(pwd)/dist/build_dockers/cross/linux.cmake
+        ctest -VV -S misc/ctest/gltest.cmake --no-compress-output
 
   test_linux_debug:
     if: (github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'update-buddy-version') != true
@@ -245,6 +248,9 @@ jobs:
     uses: ./.github/workflows/build_template.yml
     with:
       artifact_name: release_build
+      cmake_command: |
+        export CMAKE_TOOLCHAIN_FILE=$(pwd)/dist/build_dockers/cross/linux.cmake
+        ctest -VV -S misc/ctest/gltest.cmake --no-compress-output
 
   test_linux_release:
     if: (github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'update-buddy-version') != true
@@ -441,6 +447,7 @@ jobs:
         mkdir build
         cd build
         cmake -DPACK=1 ..
+        export CMAKE_TOOLCHAIN_FILE=$(pwd)/dist/build_dockers/cross/linux.cmake
         cmake --build . --target package
       cache_key: pack_jammy_x86_64
       artifact_list: "build/manticore*deb"
@@ -547,7 +554,11 @@ jobs:
     uses: ./.github/workflows/build_template.yml
     with:
       arch: aarch64
-      cmake_command: mkdir build && cd build && ctest -VV -S ../misc/ctest/justbuild.cmake -DCTEST_SOURCE_DIRECTORY=.. --no-compress-output
+      cmake_command: |
+        mkdir build
+        cd build
+        export CMAKE_TOOLCHAIN_FILE=$(pwd)/../dist/build_dockers/cross/linux.cmake
+        ctest -VV -S ../misc/ctest/justbuild.cmake -DCTEST_SOURCE_DIRECTORY=.. --no-compress-output
       cache_key: build_jammy_aarch64
 
   build_freebsd:
@@ -558,7 +569,11 @@ jobs:
     with:
       DISTR: freebsd13
       boost_url_key: none
-      cmake_command: mkdir build && cd build && ctest -VV -S ../misc/ctest/justbuild.cmake -DCTEST_SOURCE_DIRECTORY=.. --no-compress-output
+      cmake_command: |
+        mkdir build
+        cd build
+        export CMAKE_TOOLCHAIN_FILE=$(pwd)/../dist/build_dockers/cross/freebsd.cmake
+        ctest -VV -S ../misc/ctest/justbuild.cmake -DCTEST_SOURCE_DIRECTORY=.. --no-compress-output
       cache_key: build_freebsd_x86_64
 
   build_windows:
@@ -569,13 +584,12 @@ jobs:
     with:
       DISTR: windows
       arch: x64
-      sysroot_url_key: roots_mysql83_jul29
-      boost_url_key: boost80_zstd_zlib
+      sysroot_url_key: roots_mysql83_jan17
+      boost_url_key: boost_80
       CTEST_CMAKE_GENERATOR: "Ninja Multi-Config"
       CTEST_CONFIGURATION_TYPE: Debug
       cache_key: build_windows_x64
-      artifact_list: "build/xml build/src/Debug/indexer.exe build/src/Debug/searchd.exe build/src/Debug/tests.exe build/src/gtests/Debug/gmanticoretest.exe build/usr"
-
+      artifact_list: "build/xml build/src/Debug/indexer.exe build/src/Debug/searchd.exe build/src/gtests/Debug/gmanticoretest.exe build/src/Debug/*.dll build/src/gtests/Debug/*.dll build/config/*.c build/config/*.h"
   test_windows:
     if: (github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'update-buddy-version') != true
     name: Windows tests

.github/workflows/test_template.yml

@@ -15,6 +15,10 @@ on:
         required: false
         type: number
         default: 999999
+      UNITY_BUILD:
+        required: false
+        type: number
+        default: 1
       artifact_name:
         required: true
         type: string
@@ -51,6 +55,7 @@ jobs:
         # CTEST_REGEX: test_234
         WITH_COVERAGE: ${{ inputs.WITH_COVERAGE }}
         LIBS_BUNDLE:
+        UNITY_BUILD: ${{ inputs.UNITY_BUILD }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
@@ -77,7 +82,8 @@ jobs:
       - name: 🚀 Test
         id: test
         # --timeout may be not working https://gitlab.kitware.com/cmake/cmake/-/issues/23979
-        run: ctest -V -S misc/ctest/gltest.cmake --no-compress-output --timeout 600
+        run: |
+          ctest -VV -S misc/ctest/gltest.cmake --no-compress-output --timeout 600
         continue-on-error: true
 
       - name: List build files

.github/workflows/win_test_template.yml

@@ -54,13 +54,6 @@ jobs:
           path: cache
           enableCrossOsArchive: true
           key: build_windows_x64
-      - name: Copy libraries close to the binaries
-        run: |
-          cp bundle/mysql-x64/lib/libmysql* build/src/Debug/
-          cp bundle/Expat-1.95.8-x64/Libs/libexpat* build/src/Debug/
-          cp bundle/libiconv-1.9.1-x64/lib/iconv* build/src/Debug/
-      - name: Setup Flex and Bison
-        run: Install-ChocoPackage winflexbison
       - name: Run mysql
         run: |
           C:\PROGRA~1\MySQL\"MySQL Server 8.0"\bin\mysqld.exe --initialize-insecure

.gitlab-ci.yml

@@ -11,6 +11,7 @@ workflow:
 
 variables:
   VERBOSE: 1
+  UNITY_BUILD: 1
 
 stages:
   - mirroring
@@ -48,7 +49,7 @@ job_mirror:
   needs: [ ]
   tags:
     - docker-free
-  image: manticoresearch/external_toolchain:clang16_cmake3263
+  image: manticoresearch/external_toolchain:vcpkg331_20250114
   variables:
     CACHEB: "../cache"
     DIAGNOSTIC: 1 # let's set it by default forever
@@ -81,7 +82,6 @@ job_mirror:
       - build/src/indexer
       - build/src/indextool
       - build/src/searchd
-      - build/src/tests
       - build/src/gtests/gmanticoretest
       - build/config/*.c
       - build/config/*.h
@@ -125,10 +125,13 @@ job_mirror:
 
 simple_build:
   extends: [ .build ]
+  before_script:
+    - export CMAKE_TOOLCHAIN_FILE=$(pwd)/dist/build_dockers/cross/linux.cmake
   variables:
     CTEST_CMAKE_GENERATOR: "Unix Makefiles" # as Ninja fails with coverage
     CTEST_CONFIGURATION_TYPE: "Debug"
     WITH_COVERAGE: 1
+    UNITY_BUILD: 0
 
 simple_tests:
   extends: [ .tests ]
@@ -137,11 +140,14 @@ simple_tests:
     CTEST_CONFIGURATION_TYPE: "Debug"
     WITH_COVERAGE: 1
     NO_BUILD: 1
+    UNITY_BUILD: 0
 
 # ==================== Release build, then test ======================
 
 release_build:
   extends: [ .build ]
+  before_script:
+    - export CMAKE_TOOLCHAIN_FILE=$(pwd)/dist/build_dockers/cross/linux.cmake
 
 release_tests:
   extends: [ .tests ]
@@ -157,24 +163,20 @@ win_build:
   variables:
     DISTR: windows
     arch: x64
-    # use roots_mysql83_jul29 for another (non-gitlab) tests
-    sysroot: roots_jul29
-    boost: boost80_zstd_zlib
+    sysroot: roots_mysql83_jan17
+    boost: boost_80
     CTEST_CMAKE_GENERATOR: "Ninja Multi-Config"
     CTEST_CONFIGURATION_TYPE: Debug
   artifacts:
     paths:
       - build/xml
       - build/src/$CTEST_CONFIGURATION_TYPE/indexer.exe
       - build/src/$CTEST_CONFIGURATION_TYPE/searchd.exe
-      - build/src/$CTEST_CONFIGURATION_TYPE/tests.exe
-      - build/src/$CTEST_CONFIGURATION_TYPE/*.dll
       - build/src/gtests/$CTEST_CONFIGURATION_TYPE/gmanticoretest.exe
+      - build/src/$CTEST_CONFIGURATION_TYPE/*.dll
       - build/src/gtests/$CTEST_CONFIGURATION_TYPE/*.dll
-      - build/usr
-# Commented out usually need, but not on our dev where everything is installed into system
-#      - build/src/$CTEST_CONFIGURATION_TYPE/*.dll
-#      - build/src/gtests/$CTEST_CONFIGURATION_TYPE/*.dll
+      - build/config/*.c
+      - build/config/*.h
   cache:
     key: win_test
 
@@ -214,6 +216,8 @@ build_arm:
   extends: [ .build ]
   variables:
     arch: aarch64
+  before_script:
+    - export CMAKE_TOOLCHAIN_FILE=$(pwd)/dist/build_dockers/cross/linux.cmake
   script:
     - mkdir build && cd build
     - ctest -VV -S ../misc/ctest/justbuild.cmake -DCTEST_SOURCE_DIRECTORY=.. --no-compress-output
@@ -227,6 +231,8 @@ build_bsd:
   variables:
     DISTR: freebsd13
     boost: none
+  before_script:
+    - export CMAKE_TOOLCHAIN_FILE=$(pwd)/dist/build_dockers/cross/freebsd.cmake
   script:
     - mkdir build && cd build
     - ctest -VV -S ../misc/ctest/justbuild.cmake -DCTEST_SOURCE_DIRECTORY=.. --no-compress-output

CMakeLists.txt

@@ -185,10 +185,12 @@ target_compile_options ( lextra INTERFACE $<${CLANGCXX}:
 
 # Checking for unaligned RAM access
 if (CMAKE_CROSSCOMPILING)
-	if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(i.86|x86(_64)?)$"
-			OR CMAKE_SYSTEM_PROCESSOR STREQUAL amd64
-			OR CMAKE_SYSTEM_PROCESSOR STREQUAL arm64
-			OR CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64)
+	string ( TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" _PROC )
+	if (_PROC MATCHES "^(i.86|x86(_64)?)$"
+			OR _PROC STREQUAL amd64
+			OR _PROC STREQUAL arm64
+			OR _PROC STREQUAL aarch64
+			OR _PROC STREQUAL s390x)
 		set ( UNALIGNED_RAM_ACCESS_EXITCODE "0" CACHE STRING "Result from TRY_RUN" )
 		set ( UNALIGNED_RAM_ACCESS_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" )
 	endif ()
@@ -241,6 +243,8 @@ target_link_libraries ( lextra INTERFACE xxHash::xxhash )
 with_get ( jieba "Jieba" "Chinese text segmentation tool" )
 
 # Support for OpenSSL
+set ( OPENSSL_USE_STATIC_LIBS ON )
+include ( GetSSL )
 with_menu_comp ( OpenSSL SSL "OpenSSL" "for encrypted networking" )
 win_install_c ( OpenSSL SSL daemon )
 

README.md

@@ -112,7 +112,7 @@ Manticore Search was forked from [Sphinx 2.3.2](https://github.com/sphinxsearch/
   - [Sync from CSV](https://manual.manticoresearch.com/Adding_data_from_external_storages/Fetching_from_CSV,TSV#Fetching-from-TSV,CSV)
   - [With MySQL as a storage engine](https://manual.manticoresearch.com/Extensions/SphinxSE#Using-SphinxSE)
   - [With MySQL via FEDERATED engine](https://manual.manticoresearch.com/Extensions/FEDERATED)
-  - [ProxySQL](https://manticoresearch.com/2018/06/18/using-proxysql-to-route-inserts-in-a-distributed-realtime-index/)
+  - [ProxySQL](https://manticoresearch.com/blog/using-proxysql-to-route-inserts-in-a-distributed-realtime-index/)
   - [Apache Superset](https://manticoresearch.com/blog/manticoresearch-apache-superset-integration/)
   - [Grafana](https://manticoresearch.com/blog/manticoresearch-grafana-integration/)
   - [Fluentbit](https://manticoresearch.com/blog/integration-of-manticore-with-fluentbit/)

actions/update-deps/action.yml

@@ -65,10 +65,10 @@ runs:
       shell: bash
       run: |
         if [[ "${{ github.event_name }}" == "pull_request" ]]; then
-          echo "branch=buddy-${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT
-          echo "target=update-buddy-version" >> $GITHUB_OUTPUT
+          echo "branch=${{ inputs.name }}-${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT
+          echo "target=update-${{ inputs.name }}-version" >> $GITHUB_OUTPUT
         else
-          echo "branch=update-buddy-version" >> $GITHUB_OUTPUT
+          echo "branch=update-${{ inputs.name }}-version" >> $GITHUB_OUTPUT
           echo "target=master" >> $GITHUB_OUTPUT
         fi
 

api/libsphinxclient/CMakeLists.txt

@@ -20,7 +20,7 @@ PROJECT ( SPHINXCLIENT
 # our mega-lib
 add_library ( sphinxclient STATIC sphinxclient.c )
 if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "MSVC")
-	target_compile_options ( sphinxclient PRIVATE -g3 /clang:-Wall -O0 )
+	target_compile_options ( sphinxclient PRIVATE /clang:-Wall )
 else ()
 	target_compile_options ( sphinxclient PRIVATE -g3 -Wall -O0 )
 endif ()

cmake/GetColumnar.cmake

@@ -19,7 +19,7 @@ include ( update_bundle )
 
 # Versions of API headers we are need to build with.
 set ( NEED_COLUMNAR_API 26 )
-set ( NEED_SECONDARY_API 16 )
+set ( NEED_SECONDARY_API 17 )
 set ( NEED_KNN_API 3 )
 
 # Note: we don't build, neither link with columnar. Only thing we expect to get is a few interface headers, aka 'columnar_api'.
@@ -96,7 +96,7 @@ get_build ( COLUMNAR_BUILD "mcl/${AUTO_TAG}" )
 # store prev find paths to avoid polishing global scope
 backup_paths()
 
-append_prefix ( "${COLUMNAR_BUILD}" )
+prepend_prefix ( "${COLUMNAR_BUILD}" )
 
 find_package ( columnar "${NEED_API_NUMERIC_VERSION}" EXACT COMPONENTS columnar_api secondary_api knn_api CONFIG )
 return_if_all_api_found ()

cmake/GetGALERA.cmake

@@ -58,6 +58,10 @@ if (TARGET galera::galera)
 	return ()
 endif ()
 
+if (NOT BUILD_TESTING)
+	return ()
+endif ()
+
 # not found. Populate and build cache package for now and future usage.
 select_nearest_url ( GALERA_PLACE "galera" ${GALERA_BUNDLE} ${GALERA_GITHUB} )
 select_nearest_url ( WSREP_PLACE "wsrep" ${WSREP_BUNDLE} ${WSREP_GITHUB} ) # WSREP_PATH provides path to galera-imported for build