Merge branch 'main' into json_serialize

.github/actions/clp-core-build/action.yaml

@@ -8,6 +8,9 @@ inputs:
   use_published_image:
     description: "Whether to use the published container image"
     required: true
+  use_shared_libs:
+    description: "Whether to build the core binaries by linking against shared libraries"
+    required: true
   upload_binaries:
     description: "Whether to upload the core binaries"
     required: true
@@ -52,9 +55,11 @@ runs:
         --workdir /mnt/clp
         ${{steps.get_image_props.outputs.qualified_image_name}}
         bash -c "task deps:core &&
-        /mnt/clp/components/core/tools/scripts/utils/build-and-run-unit-tests.sh
-        /mnt/clp/components/core
-        /mnt/clp/components/core/build"
+        python3 /mnt/clp/components/core/tools/scripts/utils/build-and-run-unit-tests.py
+        ${{inputs.use_shared_libs == 'true' && '--use-shared-libs' || ''}}
+        --source-dir /mnt/clp/components/core
+        --build-dir /mnt/clp/components/core/build
+        --num-jobs $(getconf _NPROCESSORS_ONLN)"
       shell: "bash"
 
     - if: "inputs.upload_binaries == 'true'"

.github/workflows/clp-core-build-macos.yaml

@@ -10,7 +10,7 @@ on:
       - "components/core/tests/**"
       - "components/core/tools/scripts/lib_install/macos-12/**"
       - "components/core/tools/scripts/deps-download/**"
-      - "components/core/tools/scripts/utils/build-and-run-unit-tests.sh"
+      - "components/core/tools/scripts/utils/build-and-run-unit-tests.py"
       - "deps-tasks.yml"
       - "Taskfile.yml"
       - "tools/scripts/deps-download/**"
@@ -23,7 +23,7 @@ on:
       - "components/core/tests/**"
       - "components/core/tools/scripts/lib_install/macos-12/**"
       - "components/core/tools/scripts/deps-download/**"
-      - "components/core/tools/scripts/utils/build-and-run-unit-tests.sh"
+      - "components/core/tools/scripts/utils/build-and-run-unit-tests.py"
       - "deps-tasks.yml"
       - "Taskfile.yml"
       - "tools/scripts/deps-download/**"
@@ -36,6 +36,9 @@ concurrency:
 
 jobs:
   build-macos:
+    strategy:
+      matrix:
+        use_shared_libs: [true, false]
     runs-on: "macos-12"
     steps:
       - uses: "actions/checkout@v4"
@@ -63,4 +66,10 @@ jobs:
         shell: "bash"
         working-directory: "./components/core"
         # NOTE: We omit the Stopwatch tests since GH's macOS runner is too slow
-        run: "./tools/scripts/utils/build-and-run-unit-tests.sh . build ~[Stopwatch]"
+        run: >-
+          python3 ./tools/scripts/utils/build-and-run-unit-tests.py
+          ${{matrix.use_shared_libs == 'true' && '--use-shared-libs' || ''}}
+          --source-dir .
+          --build-dir build
+          --num-jobs $(getconf _NPROCESSORS_ONLN)
+          --test-spec "~[Stopwatch]"

.github/workflows/clp-core-build.yaml

@@ -36,7 +36,7 @@ jobs:
   filter-relevant-changes:
     runs-on: "ubuntu-latest"
     outputs:
-      centos74_image_changed: "${{steps.filter.outputs.centos74_image}}"
+      centos_stream_9_image_changed: "${{steps.filter.outputs.centos_stream_9_image}}"
       ubuntu_focal_image_changed: "${{steps.filter.outputs.ubuntu_focal_image}}"
       ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}"
       clp_changed: "${{steps.filter.outputs.clp}}"
@@ -63,12 +63,12 @@ jobs:
           # image (since it would be different from the published image).
           base: "main"
           filters: |
-            centos74_image:
+            centos_stream_9_image:
               - ".github/actions/**"
               - ".github/workflows/clp-core-build.yaml"
               - "components/core/tools/scripts/lib_install/*.sh"
-              - "components/core/tools/docker-images/clp-env-base-centos7.4/**"
-              - "components/core/tools/scripts/lib_install/centos7.4/**"
+              - "components/core/tools/docker-images/clp-env-base-centos-stream-9/**"
+              - "components/core/tools/scripts/lib_install/centos-stream-9/**"
             ubuntu_focal_image:
               - ".github/actions/**"
               - ".github/workflows/clp-core-build.yaml"
@@ -89,13 +89,13 @@ jobs:
               - "components/core/CMakeLists.txt"
               - "components/core/src/**"
               - "components/core/tests/**"
-              - "components/core/tools/scripts/utils/build-and-run-unit-tests.sh"
+              - "components/core/tools/scripts/utils/build-and-run-unit-tests.py"
               - "deps-tasks.yml"
               - "Taskfile.yml"
               - "tools/scripts/deps-download/**"
 
-  centos74-deps-image:
-    if: "needs.filter-relevant-changes.outputs.centos74_image_changed == 'true'"
+  centos-stream-9-deps-image:
+    if: "needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'true'"
     needs: "filter-relevant-changes"
     runs-on: "ubuntu-latest"
     steps:
@@ -109,7 +109,7 @@ jobs:
 
       - uses: "./.github/actions/clp-core-build-containers"
         env:
-          OS_NAME: "centos7.4"
+          OS_NAME: "centos-stream-9"
         with:
           image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}"
           docker_context: "components/core"
@@ -169,14 +169,19 @@ jobs:
             ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}}
           token: "${{secrets.GITHUB_TOKEN}}"
 
-  centos74-binaries:
+  centos-stream-9-binaries:
     # Run if the ancestor jobs succeeded OR they were skipped and clp was changed.
     if: >-
       success()
       || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true')
     needs:
-      - "centos74-deps-image"
+      - "centos-stream-9-deps-image"
       - "filter-relevant-changes"
+    strategy:
+      matrix:
+        use_shared_libs: [true, false]
+    name: "centos-stream-9-${{matrix.use_shared_libs && 'dynamic' || 'static'}}-linked-bins"
+    continue-on-error: true
     runs-on: "ubuntu-latest"
     steps:
       - uses: "actions/checkout@v4"
@@ -189,11 +194,12 @@ jobs:
 
       - uses: "./.github/actions/clp-core-build"
         env:
-          OS_NAME: "centos7.4"
+          OS_NAME: "centos-stream-9"
         with:
           image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}"
+          use_shared_libs: "${{matrix.use_shared_libs}}"
           use_published_image: >-
-            ${{needs.filter-relevant-changes.outputs.centos74_image_changed == 'false'
+            ${{needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'false'
             || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}}
           upload_binaries: "false"
 
@@ -205,6 +211,15 @@ jobs:
     needs:
       - "filter-relevant-changes"
       - "ubuntu-focal-deps-image"
+    strategy:
+      matrix:
+        include:
+          - use_shared_libs: true
+            upload_binaries: false
+          - use_shared_libs: false
+            upload_binaries: true
+    name: "ubuntu-focal-${{matrix.use_shared_libs && 'dynamic' || 'static'}}-linked-bins"
+    continue-on-error: true
     runs-on: "ubuntu-latest"
     steps:
       - uses: "actions/checkout@v4"
@@ -220,10 +235,11 @@ jobs:
           OS_NAME: "ubuntu-focal"
         with:
           image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}"
+          use_shared_libs: "${{matrix.use_shared_libs}}"
           use_published_image: >-
             ${{needs.filter-relevant-changes.outputs.ubuntu_focal_image_changed == 'false'
             || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}}
-          upload_binaries: "true"
+          upload_binaries: "${{matrix.upload_binaries}}"
           binaries_artifact_name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}"
 
   ubuntu-jammy-binaries:
@@ -234,6 +250,11 @@ jobs:
     needs:
       - "filter-relevant-changes"
       - "ubuntu-jammy-deps-image"
+    strategy:
+      matrix:
+        use_shared_libs: [true, false]
+    name: "ubuntu-jammy-${{matrix.use_shared_libs && 'dynamic' || 'static'}}-linked-bins"
+    continue-on-error: true
     runs-on: "ubuntu-latest"
     steps:
       - uses: "actions/checkout@v4"
@@ -249,6 +270,7 @@ jobs:
           OS_NAME: "ubuntu-jammy"
         with:
           image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}"
+          use_shared_libs: "${{matrix.use_shared_libs}}"
           use_published_image: >-
             ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false'
             || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}}

components/core/.clang-format

@@ -74,8 +74,8 @@ IncludeCategories:
   # NOTE: A header is grouped by first matching regex
   # Library headers. Update when adding new libraries.
   # NOTE: clang-format retains leading white-space on a line in violation of the YAML spec.
-  - Regex: "<(absl|antlr4|archive|boost|bsoncxx|catch2|curl|date|fmt|json|log_surgeon|mariadb\
-|mongocxx|msgpack|openssl|outcome|regex_utils|simdjson|spdlog|sqlite3|string_utils|yaml-cpp|zstd)"
+  - Regex: "<(absl|antlr4|archive|boost|bsoncxx|catch2|curl|date|fmt|json|log_surgeon|mongocxx\
+|msgpack|mysql|openssl|outcome|regex_utils|simdjson|spdlog|sqlite3|string_utils|yaml-cpp|zstd)"
     Priority: 3
   # C system headers
   - Regex: "^<.+\\.h>"

components/core/CMakeLists.txt

@@ -51,10 +51,20 @@ endif()
 
 # Detect linking mode (static or shared); Default to static.
 set(CLP_USE_STATIC_LIBS ON CACHE BOOL "Whether to link against static libraries")
-if (CLP_USE_STATIC_LIBS AND APPLE)
-    message(AUTHOR_WARNING "Building with static libraries is unsupported on macOS."
-            " Switching to shared libraries.")
-    set(CLP_USE_STATIC_LIBS OFF)
+if (CLP_USE_STATIC_LIBS)
+    if (APPLE)
+        set(CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM "macOS")
+    elseif (EXISTS "/etc/centos-release")
+        set(CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM "CentOS")
+    endif()
+
+    if (DEFINED CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM)
+        message(
+            AUTHOR_WARNING
+            "Building with static libraries is unsupported on"
+            " ${CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM}. Switching to shared libraries.")
+        set(CLP_USE_STATIC_LIBS OFF)
+    endif()
 endif ()
 if(CLP_USE_STATIC_LIBS)
     set(CLP_LIBS_STRING "static")
@@ -198,6 +208,8 @@ add_subdirectory(submodules/abseil-cpp EXCLUDE_FROM_ALL)
 # Add simdjson
 add_subdirectory(submodules/simdjson EXCLUDE_FROM_ALL)
 
+find_package(Threads REQUIRED)
+
 # Add yaml-cpp
 add_subdirectory(submodules/yaml-cpp EXCLUDE_FROM_ALL)
 

components/core/src/clp/MySQLDB.hpp

@@ -3,7 +3,7 @@
 
 #include <string>
 
-#include <mariadb/mysql.h>
+#include <mysql.h>
 
 #include "Defs.h"
 #include "ErrorCode.hpp"

components/core/src/clp/MySQLParamBindings.hpp

@@ -4,7 +4,7 @@
 #include <cstdint>
 #include <vector>
 
-#include <mariadb/mysql.h>
+#include <mysql.h>
 
 #include "ErrorCode.hpp"
 #include "TraceableException.hpp"

components/core/src/clp/MySQLPreparedStatement.hpp

@@ -4,7 +4,7 @@
 #include <string>
 #include <vector>
 
-#include <mariadb/mysql.h>
+#include <mysql.h>
 
 #include "ErrorCode.hpp"
 #include "MySQLParamBindings.hpp"

components/core/src/glt/MySQLDB.hpp

@@ -3,7 +3,7 @@
 
 #include <string>
 
-#include <mariadb/mysql.h>
+#include <mysql.h>
 
 #include "Defs.h"
 #include "ErrorCode.hpp"

components/core/src/glt/MySQLParamBindings.hpp

@@ -4,7 +4,7 @@
 #include <cstdint>
 #include <vector>
 
-#include <mariadb/mysql.h>
+#include <mysql.h>
 
 #include "ErrorCode.hpp"
 #include "TraceableException.hpp"

components/core/src/glt/MySQLPreparedStatement.hpp

@@ -4,7 +4,7 @@
 #include <string>
 #include <vector>
 
-#include <mariadb/mysql.h>
+#include <mysql.h>
 
 #include "ErrorCode.hpp"
 #include "MySQLParamBindings.hpp"

components/core/src/reducer/CMakeLists.txt

@@ -44,11 +44,16 @@ target_include_directories(reducer-server PRIVATE "${PROJECT_SOURCE_DIR}/submodu
 target_link_libraries(reducer-server
         PRIVATE
         Boost::program_options
+        Boost::system
         clp::string_utils
         fmt::fmt
         ${MONGOCXX_TARGET}
         msgpack-cxx
         spdlog::spdlog
+        # Threads::Threads is necessary because on Ubuntu Focal (20.04), libssl (a dependency of
+        # libmongocxx) depends on libpthread, but its cmake script doesn't seem to provide the flags
+        # to link against libpthread.
+        Threads::Threads
 )
 # Put the built executable at the root of the build directory
 set_target_properties(

components/core/tools/docker-images/clp-env-base-centos-stream-9/Dockerfile

@@ -0,0 +1,22 @@
+FROM dokken/centos-stream-9 AS base
+
+WORKDIR /root
+
+RUN mkdir -p ./tools/scripts/lib_install
+ADD ./tools/scripts/lib_install ./tools/scripts/lib_install
+
+RUN ./tools/scripts/lib_install/centos-stream-9/install-all.sh
+
+# NOTE:
+# 1. `task` doesn't have an apt/dnf package so we use its install script.
+# 2. We don't want to install it using `install-prebuilt-packages.sh` since users may use that on
+#    their own machines and it would change their environment in a way that can't easily be undone.
+RUN sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /usr/local/bin
+
+# Remove cached files
+RUN dnf clean all \
+    && rm -rf /tmp/* /var/tmp/*
+
+# Flatten the image
+FROM scratch
+COPY --from=base / /

components/core/tools/docker-images/clp-env-base-centos-stream-9/build.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+# Exit on any error
+set -e
+
+# Error on undefined variable
+set -u
+
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+component_root="${script_dir}/../../../"
+
+docker build \
+    --tag clp-core-dependencies-x86-centos-stream-9:dev \
+    "$component_root" \
+    --file "${script_dir}/Dockerfile"