Build confluent-kafka with kerberos support

While leaving cyrus-sasl out of the wheel like in Linux

.builders/images/macos/build.sh

@@ -23,28 +23,54 @@ export DYLD_LIBRARY_PATH="${DD_PREFIX_PATH}/lib:${DYLD_LIBRARY_PATH:-}"
 
 "${DD_PYTHON3}" -m pip install --no-warn-script-location -r "runner_dependencies.txt"
 
+# Install always with our own prefix path
+install-from-source() {
+    bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" "$@"
+}
+
 # Restore cache if it exists
 if [[ -n ${DD_PREFIX_CACHE:-} && -d ${DD_PREFIX_CACHE:-} ]]; then
     cp -r "${DD_PREFIX_CACHE}" "${DD_PREFIX_PATH}"
 else
     # openssl
     DOWNLOAD_URL="https://www.openssl.org/source/openssl-{{version}}.tar.gz" \
-                VERSION="3.0.12" \
-                SHA256="f93c9e8edde5e9166119de31755fc87b4aa34863662f67ddfcba14d0b6b69b61" \
-                RELATIVE_PATH="openssl-{{version}}" \
-                CONFIGURE_SCRIPT="./config" \
-                bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" \
-                -fPIC shared \
-                no-module \
-                no-comp no-idea no-mdc2 no-rc5 no-ssl3 no-gost
+    VERSION="3.0.12" \
+    SHA256="f93c9e8edde5e9166119de31755fc87b4aa34863662f67ddfcba14d0b6b69b61" \
+    RELATIVE_PATH="openssl-{{version}}" \
+    CONFIGURE_SCRIPT="./config" \
+      install-from-source \
+        -fPIC shared \
+        no-module \
+        no-comp no-idea no-mdc2 no-rc5 no-ssl3 no-gost
 
     # postgresql
     DOWNLOAD_URL="https://ftp.postgresql.org/pub/source/v{{version}}/postgresql-{{version}}.tar.bz2" \
-                VERSION="16.0" \
-                SHA256="df9e823eb22330444e1d48e52cc65135a652a6fdb3ce325e3f08549339f51b99" \
-                RELATIVE_PATH="postgresql-{{version}}" \
-                bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" \
-                --with-openssl --without-readline  --without-icu
+    VERSION="16.0" \
+    SHA256="df9e823eb22330444e1d48e52cc65135a652a6fdb3ce325e3f08549339f51b99" \
+    RELATIVE_PATH="postgresql-{{version}}" \
+      install-from-source \
+        --with-openssl --without-readline  --without-icu
+
+    # Dependencies needed to build librdkafka (and thus, confluent-kafka) with kerberos support
+    # Note that we don't ship these but rely on the Agent providing a working cyrus-sasl installation
+    # with kerberos support, therefore we only need to watch out for the version of cyrus-sasl being
+    # compatible with that in the Agent, the rest shouldn't matter much
+    DOWNLOAD_URL="https://github.com/LMDB/lmdb/archive/LMDB_{{version}}.tar.gz" \
+    VERSION="0.9.29" \
+    SHA256="22054926b426c66d8f2bc22071365df6e35f3aacf19ad943bc6167d4cae3bebb" \
+    RELATIVE_PATH="lmdb-LMDB_{{version}}/libraries/liblmdb" \
+    CONFIGURE_SCRIPT="true" \
+      install-from-source
+    DOWNLOAD_URL="https://mirrors.edge.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v{{version}}/e2fsprogs-{{version}}.tar.gz" \
+    VERSION="1.47.0" \
+    SHA256="0b4fe723d779b0927fb83c9ae709bc7b40f66d7df36433bef143e41c54257084" \
+    RELATIVE_PATH="e2fsprogs-{{version}}" \
+      install-from-source
+    DOWNLOAD_URL="https://github.com/cyrusimap/cyrus-sasl/releases/download/cyrus-sasl-{{version}}/cyrus-sasl-{{version}}.tar.gz" \
+    VERSION="2.1.28" \
+    SHA256="7ccfc6abd01ed67c1a0924b353e526f1b766b21f42d4562ee635a8ebfc5bb38c" \
+    RELATIVE_PATH="cyrus-sasl-{{version}}" \
+      install-from-source --with-dblib=lmdb --enable-gssapi="${DD_PREFIX_PATH}" --disable-macos-framework
 
     # Cache everything under prefix
     if [[ -n ${DD_PREFIX_CACHE:-} ]]; then

.builders/images/macos/extra_build.sh

@@ -11,5 +11,5 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then
       VERSION="${kafka_version}" \
       SHA256="2d49c35c77eeb3d42fa61c43757fcbb6a206daa560247154e60642bcdcc14d12" \
       RELATIVE_PATH="librdkafka-{{version}}" \
-      bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" # --enable-sasl
+      bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" --enable-sasl --enable-curl
 fi

.builders/scripts/repair_wheels.py

@@ -123,6 +123,10 @@ def repair_darwin(source_dir: str, output_dir: str) -> None:
     exclusions = [
         # pymqi
         'pymqe.cpython-311-darwin.so',
+        # confluent_kafka
+        # We leave cyrus-sasl out of the wheel because of the complexity involved in bundling it portably.
+        # This means the confluent-kafka wheel will have a runtime dependency on this library
+        'libsasl2.3.dylib',
     ]
 
     def copy_filt_func(libname):
@@ -138,7 +142,6 @@ def copy_filt_func(libname):
         delocate_wheel(
             str(wheel),
             os.path.join(output_dir, os.path.basename(wheel)),
-            lib_sdir='.libs',
             copy_filt_func=copy_filt_func,
         )
         print('Repaired wheel')