Add MacOS dependency build

.builders/build.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import argparse
+import os
 import shutil
 import subprocess
 import sys
@@ -65,7 +66,83 @@ def read_dependencies() -> dict[str, list[str]]:
 
 
 def build_macos():
-    sys.exit('macOS is not supported')
+    parser = argparse.ArgumentParser(prog='builder', allow_abbrev=False)
+    parser.add_argument('output_dir')
+    parser.add_argument('--python', default='3')
+    parser.add_argument('--builder-root', required=True,
+                        help='Path to a folder where things will be installed during builder setup.')
+    parser.add_argument('--skip-setup', default=False, action='store_true',
+                        help='Skip builder setup, assuming it has already been set up.')
+    args = parser.parse_args()
+
+    context_path = HERE / 'images' / 'macos'
+    builder_root = Path(args.builder_root).absolute()
+    builder_root.mkdir(exist_ok=True)
+
+    with temporary_directory() as temp_dir:
+        mount_dir = temp_dir / 'mnt'
+        mount_dir.mkdir()
+
+        build_context_dir = shutil.copytree(context_path, mount_dir / 'build_context', dirs_exist_ok=True)
+        # Copy utilities shared by multiple images
+        for entry in context_path.parent.iterdir():
+            if entry.is_file():
+                shutil.copy2(entry, build_context_dir)
+
+        # Folders required by the build_wheels script
+        wheels_dir = mount_dir / 'wheels'
+        wheels_dir.mkdir()
+        built_wheels_dir = wheels_dir / 'built'
+        built_wheels_dir.mkdir()
+        external_wheels_dir = wheels_dir / 'external'
+        external_wheels_dir.mkdir()
+
+        dependency_file = mount_dir / 'requirements.in'
+        dependency_file.write_text('\n'.join(chain.from_iterable(read_dependencies().values())))
+        shutil.copy(HERE / 'deps' / 'build_dependencies.txt', mount_dir)
+        shutil.copytree(HERE / 'scripts', mount_dir / 'scripts')
+        shutil.copytree(HERE / 'patches', mount_dir / 'patches')
+
+        prefix_path = builder_root / 'prefix'
+        env = {
+            **os.environ,
+            'DD_MOUNT_DIR': mount_dir,
+            # Paths to pythons
+            'DD_PY3_BUILDENV_PATH': builder_root / 'py3' / 'bin' / 'python',
+            'DD_PY2_BUILDENV_PATH': builder_root / 'py2' / 'bin' / 'python',
+            # Path where we'll install libraries that we build
+            'DD_PREFIX_PATH': prefix_path,
+            # Common compilation flags
+            'LDFLAGS': f'-L{prefix_path}/lib',
+            'CFLAGS': f'-I{prefix_path}/include -O2',
+            # Build command for extra platform-specific build steps
+            'DD_BUILD_COMMAND': f'bash {build_context_dir}/extra_build.sh'
+        }
+
+        if not args.skip_setup:
+            check_process(
+                ['bash', str(HERE / 'images' / 'macos' / 'builder_setup.sh')],
+                env=env,
+                cwd=builder_root,
+            )
+
+        check_process(
+            [os.environ['DD_PYTHON3'], str(mount_dir / 'scripts' / 'build_wheels.py'), '--python', args.python],
+            env=env,
+            cwd=builder_root,
+        )
+
+        output_dir = Path(args.output_dir)
+        if output_dir.is_dir():
+            shutil.rmtree(output_dir)
+
+        # Move wheels to the output directory
+        wheels_dir = mount_dir / 'wheels'
+        shutil.move(wheels_dir, output_dir / 'wheels')
+
+        # Move the final requirements file to the output directory
+        final_requirements = mount_dir / 'frozen.txt'
+        shutil.move(final_requirements, output_dir)
 
 
 def build_image():

.builders/images/linux-x86_64/install-from-source.sh → .builders/images/install-from-source.sh

@@ -10,8 +10,7 @@
 # Optional:
 # - CONFIGURE_SCRIPT: Alternative to the default ./configure
 
-
-set -exu
+set -euxo pipefail
 
 url=${DOWNLOAD_URL//'{{version}}'/${VERSION}}
 relative_path=${RELATIVE_PATH//'{{version}}'/${VERSION}}

.builders/images/linux-x86_64/Dockerfile

@@ -21,6 +21,7 @@ ENV LD_LIBRARY_PATH="/opt/python/${PYTHON2_VERSION}/lib:${LD_LIBRARY_PATH}"
 # Set up virtual environment for Python 2
 RUN /opt/python/${PYTHON2_VERSION}/bin/python -m pip install --no-warn-script-location virtualenv \
  && /opt/python/${PYTHON2_VERSION}/bin/python -m virtualenv /py2
+ENV DD_PY2_BUILDENV_PATH=/py2/bin/python
 
 # openssl
 RUN yum install -y perl-IPC-Cmd && \
@@ -50,6 +51,7 @@ ENV LD_LIBRARY_PATH="/opt/python/${PYTHON_VERSION}/lib:${LD_LIBRARY_PATH}"
 RUN /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install --no-warn-script-location --upgrade pip \
  && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install --no-warn-script-location virtualenv \
  && /opt/python/${PYTHON_VERSION}/bin/python3 -m virtualenv /py3
+ENV DD_PY3_BUILDENV_PATH=/py3/bin/python
 
 # Rust toolchain
 ENV RUST_VERSION="nightly-2022-05-15"
@@ -165,5 +167,6 @@ RUN python3 -m pip install --no-warn-script-location -r /runner_dependencies.txt
 
 COPY build_script.sh /build_script.sh
 ENV DD_BUILD_COMMAND="bash /build_script.sh"
+ENV DD_MOUNT_DIR="/home"
 
 ENTRYPOINT ["python3", "/home/scripts/build_wheels.py"]

.builders/images/macos/builder_setup.sh

@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+"${DD_PYTHON3}" -m pip install --no-warn-script-location --upgrade pip
+"${DD_PYTHON3}" -m pip install --no-warn-script-location virtualenv
+"${DD_PYTHON3}" -m virtualenv py3
+
+"${DD_PYTHON2}" -m pip install --no-warn-script-location --upgrade pip
+"${DD_PYTHON2}" -m pip install --no-warn-script-location virtualenv
+"${DD_PYTHON2}" -m virtualenv py2
+
+"${DD_PYTHON3}" -m pip install --no-warn-script-location -r "${DD_MOUNT_DIR}/build_context/runner_dependencies.txt"
+
+# Install always with our own prefix path
+cp "${DD_MOUNT_DIR}/build_context/install-from-source.sh" .
+install-from-source() {
+    bash "install-from-source.sh" --prefix="${DD_PREFIX_PATH}" "$@"
+}
+
+# mqi
+IBM_MQ_VERSION=9.2.4.0-IBM-MQ-DevToolkit
+curl --retry 5 --fail "https://s3.amazonaws.com/dd-agent-omnibus/ibm-mq-backup/${IBM_MQ_VERSION}-MacX64.pkg" -o /tmp/mq_client.pkg
+sudo installer -pkg /tmp/mq_client.pkg -target /
+rm -rf /tmp/mq_client.pkg
+
+# openssl
+DOWNLOAD_URL="https://www.openssl.org/source/openssl-{{version}}.tar.gz" \
+VERSION="3.0.12" \
+SHA256="f93c9e8edde5e9166119de31755fc87b4aa34863662f67ddfcba14d0b6b69b61" \
+RELATIVE_PATH="openssl-{{version}}" \
+CONFIGURE_SCRIPT="./config" \
+  install-from-source \
+    -fPIC shared \
+    no-module \
+    no-comp no-idea no-mdc2 no-rc5 no-ssl3 no-gost
+
+# libxml & libxslt for lxml
+DOWNLOAD_URL="https://download.gnome.org/sources/libxml2/2.10/libxml2-{{version}}.tar.xz" \
+VERSION="2.10.3" \
+SHA256="5d2cc3d78bec3dbe212a9d7fa629ada25a7da928af432c93060ff5c17ee28a9c" \
+RELATIVE_PATH="libxml2-{{version}}" \
+  install-from-source \
+    --without-iconv \
+    --without-python \
+    --without-icu \
+    --without-debug \
+    --without-mem-debug \
+    --without-run-debug \
+    --without-legacy \
+    --without-catalog \
+    --without-docbook \
+    --disable-static
+
+DOWNLOAD_URL="https://download.gnome.org/sources/libxslt/1.1/libxslt-{{version}}.tar.xz" \
+VERSION="1.1.37" \
+SHA256="3a4b27dc8027ccd6146725950336f1ec520928f320f144eb5fa7990ae6123ab4" \
+RELATIVE_PATH="libxslt-{{version}}" \
+  install-from-source \
+    --with-libxml-prefix="${DD_PREFIX_PATH}" \
+    --without-python \
+    --without-crypto \
+    --without-profiler \
+    --without-debugger \
+    --disable-static
+
+# curl
+DOWNLOAD_URL="https://curl.haxx.se/download/curl-{{version}}.tar.gz" \
+VERSION="8.4.0" \
+SHA256="816e41809c043ff285e8c0f06a75a1fa250211bbfb2dc0a037eeef39f1a9e427" \
+RELATIVE_PATH="curl-{{version}}" \
+  install-from-source \
+    --disable-manual \
+    --disable-debug \
+    --enable-optimize \
+    --disable-static \
+    --disable-ldap \
+    --disable-ldaps \
+    --disable-rtsp \
+    --enable-proxy \
+    --disable-dependency-tracking \
+    --enable-ipv6 \
+    --without-libidn \
+    --without-gnutls \
+    --without-librtmp \
+    --without-libssh2 \
+    --with-ssl="${DD_PREFIX_PATH}"
+# Remove the binary installed so that we consistenly use the same original `curl` binary
+rm "${DD_PREFIX_PATH}/bin/curl"
+
+# Dependencies needed to build librdkafka (and thus, confluent-kafka) with kerberos support
+# Note that we don't ship these but rely on the Agent providing a working cyrus-sasl installation
+# with kerberos support, therefore we only need to watch out for the version of cyrus-sasl being
+# compatible with that in the Agent, the rest shouldn't matter much
+DOWNLOAD_URL="https://github.com/LMDB/lmdb/archive/LMDB_{{version}}.tar.gz" \
+VERSION="0.9.29" \
+SHA256="22054926b426c66d8f2bc22071365df6e35f3aacf19ad943bc6167d4cae3bebb" \
+RELATIVE_PATH="lmdb-LMDB_{{version}}/libraries/liblmdb" \
+CONFIGURE_SCRIPT="true" \
+  install-from-source
+DOWNLOAD_URL="https://github.com/cyrusimap/cyrus-sasl/releases/download/cyrus-sasl-{{version}}/cyrus-sasl-{{version}}.tar.gz" \
+VERSION="2.1.28" \
+SHA256="7ccfc6abd01ed67c1a0924b353e526f1b766b21f42d4562ee635a8ebfc5bb38c" \
+RELATIVE_PATH="cyrus-sasl-{{version}}" \
+  install-from-source --with-dblib=lmdb --enable-gssapi="${DD_PREFIX_PATH}" --disable-macos-framework

.builders/images/macos/extra_build.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+set -exu
+
+# Packages which must be built from source
+always_build=()
+
+if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then
+    # confluent-kafka and librdkafka need to be compiled from source to get kerberos support
+    # The librdkafka version needs to stay in sync with the confluent-kafka version,
+    # thus we extract the version from the requirements file.
+    kafka_version=$(grep 'confluent-kafka==' "${DD_MOUNT_DIR}/requirements.in" | sed -E 's/^.*([[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+).*$/\1/')
+    DOWNLOAD_URL="https://github.com/confluentinc/librdkafka/archive/refs/tags/v{{version}}.tar.gz" \
+      VERSION="${kafka_version}" \
+      SHA256="2d49c35c77eeb3d42fa61c43757fcbb6a206daa560247154e60642bcdcc14d12" \
+      RELATIVE_PATH="librdkafka-{{version}}" \
+      bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" --enable-sasl --enable-curl
+
+    always_build+=("confluent-kafka")
+fi
+
+# Empty arrays are flagged as unset when using the `-u` flag. This is the safest way to work around that
+# (see https://stackoverflow.com/a/61551944)
+pip_no_binary=${always_build[@]+"${always_build[@]}"}
+if [[ "$pip_no_binary" ]]; then
+    # If there are any packages that must always be built, inform pip
+    echo "PIP_NO_BINARY=\"${pip_no_binary}\"" >> $DD_ENV_FILE
+fi

.builders/images/runner_dependencies.txt

@@ -2,3 +2,4 @@ python-dotenv==1.0.0
 urllib3==2.1.0
 auditwheel==5.4.0; sys_platform == 'linux'
 delvewheel==1.5.2; sys_platform == 'win32'
+delocate==0.10.7; sys_platform == 'darwin'

.builders/scripts/build_wheels.py

@@ -26,10 +26,10 @@ def path_to_uri(path: str) -> str:
 else:
     import shlex
 
-    PY3_PATH = Path('/py3/bin/python')
-    PY2_PATH = Path('/py2/bin/python')
-    MOUNT_DIR = Path('/home')
-    ENV_FILE = Path('/.env')
+    PY3_PATH = Path(os.environ.get('DD_PY3_BUILDENV_PATH', '/py3/bin/python'))
+    PY2_PATH = Path(os.environ.get('DD_PY2_BUILDENV_PATH', '/py2/bin/python'))
+    MOUNT_DIR = Path(os.environ.get('DD_MOUNT_DIR', '/home'))
+    ENV_FILE = MOUNT_DIR / '.env'
 
     def join_command_args(args: list[str]) -> str:
         return shlex.join(args)

.builders/scripts/repair_wheels.py

@@ -116,9 +116,67 @@ def repair_windows(source_dir: str, built_dir: str, external_dir: str) -> None:
             sys.exit(process.returncode)
 
 
+def repair_darwin(source_dir: str, built_dir: str, external_dir: str) -> None:
+    from delocate import delocate_wheel
+    exclusions = [re.compile(s) for s in [
+        # pymqi
+        r'pymqe\.cpython-\d+-darwin\.so',
+        # confluent_kafka
+        # We leave cyrus-sasl out of the wheel because of the complexity involved in bundling it portably.
+        # This means the confluent-kafka wheel will have a runtime dependency on this library
+        r'libsasl2.\d\.dylib',
+        # Whitelisted libraries based on the health check default whitelist that we have on omnibus:
+        # https://github.com/DataDog/omnibus-ruby/blob/044a81fa1b0f1c50fc7083cb45e7d8f90d96905b/lib/omnibus/health_check.rb#L133-L152
+        # We use that instead of the more relaxed policy that delocate_wheel defaults to.
+        r'libobjc\.A\.dylib',
+        r'libSystem\.B\.dylib',
+        # Symlink of the previous one
+        r'libgcc_s\.1\.dylib',
+        r'CoreFoundation',
+        r'CoreServices',
+        r'Tcl$',
+        r'Cocoa$',
+        r'Carbon$',
+        r'IOKit$',
+        r'Kerberos',
+        r'Tk$',
+        r'libutil\.dylib',
+        r'libffi\.dylib',
+        r'libncurses\.5\.4\.dylib',
+        r'libiconv',
+        r'libstdc\+\+\.6\.dylib',
+        r'libc\+\+\.1\.dylib',
+        r'^/System/Library/',
+        r'libz\.1\.dylib',
+    ]]
+
+    def copy_filt_func(libname):
+        return not any(excl.search(libname) for excl in exclusions)
+
+    for wheel in iter_wheels(source_dir):
+        print(f'--> {wheel.name}')
+        if not wheel_was_built(wheel):
+            print('Using existing wheel')
+            shutil.move(wheel, external_dir)
+            continue
+
+        copied_libs = delocate_wheel(
+            str(wheel),
+            os.path.join(built_dir, os.path.basename(wheel)),
+            copy_filt_func=copy_filt_func,
+        )
+        print('Repaired wheel')
+        if copied_libs:
+            print('Libraries copied into the wheel:')
+            print('\n'.join(copied_libs))
+        else:
+            print('No libraries were copied into the wheel.')
+
+
 REPAIR_FUNCTIONS = {
     'linux': repair_linux,
     'win32': repair_windows,
+    'darwin': repair_darwin,
 }
 
 

.github/workflows/build-deps.yml

@@ -98,3 +98,42 @@ jobs:
     - name: Publish image
       if: github.event_name == 'push' && steps.changed-files.outputs.builders_any_changed == 'true'
       run: docker push ghcr.io/datadog/agent-int-builder:${{ matrix.job.image }}
+
+  build-deps-macos:
+    name: Target macOS
+    runs-on: macos-12
+    steps:
+      - name: Set up environment
+        run: |
+          # We remove everything that comes pre-installed via brew to avoid depending or shipping stuff that
+          # comes in the runner through brew to better control what might get shipped in the wheels via `delocate`
+          brew remove --force --ignore-dependencies $(brew list --formula)
+          brew install coreutils
+
+      - name: Set up Python
+        env:
+          # Despite the name, this is built for the macOS 11 SDK on arm64 and 10.9+ on intel
+          PYTHON3_DOWNLOAD_URL: "https://www.python.org/ftp/python/3.11.5/python-3.11.5-macos11.pkg"
+          PYTHON2_DOWNLOAD_URL: "https://www.python.org/ftp/python/2.7.18/python-2.7.18-macosx10.9.pkg"
+        run: |
+          curl "$PYTHON3_DOWNLOAD_URL" -o python3.pkg
+          sudo installer -pkg python3.pkg -target /
+
+          curl "$PYTHON2_DOWNLOAD_URL" -o python2.pkg
+          sudo installer -pkg python2.pkg -target /
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run the build
+        env:
+          DD_PYTHON3: "/Library/Frameworks/Python.framework/Versions/3.11/bin/python3"
+          DD_PYTHON2: "/Library/Frameworks/Python.framework/Versions/2.7/bin/python"
+          # This sets the minimum mac os version compatible for all built artifacts
+          MACOSX_DEPLOYMENT_TARGET: "10.12"
+        run: |
+          ${DD_PYTHON3} -m pip install -r .builders/deps/host_dependencies.txt
+
+          mkdir builder_root
+          ${DD_PYTHON3} .builders/build.py --builder-root builder_root --python 3 out_py3
+          ${DD_PYTHON3} .builders/build.py --builder-root builder_root --skip-setup --python 2 out_py2