Add workflow to test build on CI

DataDog · Jan 18, 2024 · bcdb96e · bcdb96e
1 parent 5abf1c6
commit bcdb96e
Show file tree

Hide file tree

Showing 3 changed files with 131 additions and 85 deletions.
diff --git a/.builders/build.py b/.builders/build.py
@@ -93,7 +93,7 @@ def build_macos():
 
         env = {
             **os.environ,
-            'DD_PREFIX_CACHE': args.cache_dir,
+            'DD_PREFIX_CACHE': args.cache_dir or '',
             'DD_MOUNT_DIR': mount_dir,
         }
         check_process(

diff --git a/.builders/images/macos/build.sh b/.builders/images/macos/build.sh
@@ -30,94 +30,101 @@ install-from-source() {
     bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" "$@"
 }
 
+# mqi
+IBM_MQ_VERSION=9.2.4.0-IBM-MQ-DevToolkit
+curl --retry 5 --fail "https://s3.amazonaws.com/dd-agent-omnibus/ibm-mq-backup/${IBM_MQ_VERSION}-MacX64.pkg" -o /tmp/mq_client.pkg
+sudo installer -pkg /tmp/mq_client.pkg -target /
+rm -rf /tmp/mq_client.pkg
+
 # Restore cache if it exists
 if [[ -n ${DD_PREFIX_CACHE:-} && -d ${DD_PREFIX_CACHE:-} ]]; then
+    echo "Using provided cache for built libraries."
     cp -r "${DD_PREFIX_CACHE}" "${DD_PREFIX_PATH}"
 else
-    # openssl
-    DOWNLOAD_URL="https://www.openssl.org/source/openssl-{{version}}.tar.gz" \
-    VERSION="3.0.12" \
-    SHA256="f93c9e8edde5e9166119de31755fc87b4aa34863662f67ddfcba14d0b6b69b61" \
-    RELATIVE_PATH="openssl-{{version}}" \
-    CONFIGURE_SCRIPT="./config" \
-      install-from-source \
-        -fPIC shared \
-        no-module \
-        no-comp no-idea no-mdc2 no-rc5 no-ssl3 no-gost
-
-    # libxml & libxslt for lxml
-    DOWNLOAD_URL="https://download.gnome.org/sources/libxml2/2.10/libxml2-{{version}}.tar.xz" \
-    VERSION="2.10.3" \
-    SHA256="5d2cc3d78bec3dbe212a9d7fa629ada25a7da928af432c93060ff5c17ee28a9c" \
-    RELATIVE_PATH="libxml2-{{version}}" \
-      install-from-source \
-        --without-iconv \
-        --without-python \
-        --without-icu \
-        --without-debug \
-        --without-mem-debug \
-        --without-run-debug \
-        --without-legacy \
-        --without-catalog \
-        --without-docbook \
-        --disable-static
-
-    DOWNLOAD_URL="https://download.gnome.org/sources/libxslt/1.1/libxslt-{{version}}.tar.xz" \
-    VERSION="1.1.37" \
-    SHA256="3a4b27dc8027ccd6146725950336f1ec520928f320f144eb5fa7990ae6123ab4" \
-    RELATIVE_PATH="libxslt-{{version}}" \
-      install-from-source \
-        --with-libxml-prefix="${DD_PREFIX_PATH}" \
-        --without-python \
-        --without-crypto \
-        --without-profiler \
-        --without-debugger \
-        --disable-static
-
-    # curl
-    DOWNLOAD_URL="https://curl.haxx.se/download/curl-{{version}}.tar.gz" \
-    VERSION="8.4.0" \
-    SHA256="816e41809c043ff285e8c0f06a75a1fa250211bbfb2dc0a037eeef39f1a9e427" \
-    RELATIVE_PATH="curl-{{version}}" \
-      install-from-source \
-        --disable-manual \
-        --disable-debug \
-        --enable-optimize \
-        --disable-static \
-        --disable-ldap \
-        --disable-ldaps \
-        --disable-rtsp \
-        --enable-proxy \
-        --disable-dependency-tracking \
-        --enable-ipv6 \
-        --without-libidn \
-        --without-gnutls \
-        --without-librtmp \
-        --without-libssh2 \
-        --with-ssl="${DD_PREFIX_PATH}"
-    # Remove the binary installed so that we consistenly use the same original `curl` binary
-    rm "${DD_PREFIX_PATH}/bin/curl"
-
-    # Dependencies needed to build librdkafka (and thus, confluent-kafka) with kerberos support
-    # Note that we don't ship these but rely on the Agent providing a working cyrus-sasl installation
-    # with kerberos support, therefore we only need to watch out for the version of cyrus-sasl being
-    # compatible with that in the Agent, the rest shouldn't matter much
-    DOWNLOAD_URL="https://github.com/LMDB/lmdb/archive/LMDB_{{version}}.tar.gz" \
-    VERSION="0.9.29" \
-    SHA256="22054926b426c66d8f2bc22071365df6e35f3aacf19ad943bc6167d4cae3bebb" \
-    RELATIVE_PATH="lmdb-LMDB_{{version}}/libraries/liblmdb" \
-    CONFIGURE_SCRIPT="true" \
-      install-from-source
-    DOWNLOAD_URL="https://github.com/cyrusimap/cyrus-sasl/releases/download/cyrus-sasl-{{version}}/cyrus-sasl-{{version}}.tar.gz" \
-    VERSION="2.1.28" \
-    SHA256="7ccfc6abd01ed67c1a0924b353e526f1b766b21f42d4562ee635a8ebfc5bb38c" \
-    RELATIVE_PATH="cyrus-sasl-{{version}}" \
-      install-from-source --with-dblib=lmdb --enable-gssapi="${DD_PREFIX_PATH}" --disable-macos-framework
-
-    # Cache everything under prefix
-    if [[ -n ${DD_PREFIX_CACHE:-} ]]; then
-        cp -r "${DD_PREFIX_PATH}" "${DD_PREFIX_CACHE}"
-    fi
+    # # openssl
+    # DOWNLOAD_URL="https://www.openssl.org/source/openssl-{{version}}.tar.gz" \
+    # VERSION="3.0.12" \
+    # SHA256="f93c9e8edde5e9166119de31755fc87b4aa34863662f67ddfcba14d0b6b69b61" \
+    # RELATIVE_PATH="openssl-{{version}}" \
+    # CONFIGURE_SCRIPT="./config" \
+    #   install-from-source \
+    #     -fPIC shared \
+    #     no-module \
+    #     no-comp no-idea no-mdc2 no-rc5 no-ssl3 no-gost
+
+    # # libxml & libxslt for lxml
+    # DOWNLOAD_URL="https://download.gnome.org/sources/libxml2/2.10/libxml2-{{version}}.tar.xz" \
+    # VERSION="2.10.3" \
+    # SHA256="5d2cc3d78bec3dbe212a9d7fa629ada25a7da928af432c93060ff5c17ee28a9c" \
+    # RELATIVE_PATH="libxml2-{{version}}" \
+    #   install-from-source \
+    #     --without-iconv \
+    #     --without-python \
+    #     --without-icu \
+    #     --without-debug \
+    #     --without-mem-debug \
+    #     --without-run-debug \
+    #     --without-legacy \
+    #     --without-catalog \
+    #     --without-docbook \
+    #     --disable-static
+
+    # DOWNLOAD_URL="https://download.gnome.org/sources/libxslt/1.1/libxslt-{{version}}.tar.xz" \
+    # VERSION="1.1.37" \
+    # SHA256="3a4b27dc8027ccd6146725950336f1ec520928f320f144eb5fa7990ae6123ab4" \
+    # RELATIVE_PATH="libxslt-{{version}}" \
+    #   install-from-source \
+    #     --with-libxml-prefix="${DD_PREFIX_PATH}" \
+    #     --without-python \
+    #     --without-crypto \
+    #     --without-profiler \
+    #     --without-debugger \
+    #     --disable-static
+
+    # # curl
+    # DOWNLOAD_URL="https://curl.haxx.se/download/curl-{{version}}.tar.gz" \
+    # VERSION="8.4.0" \
+    # SHA256="816e41809c043ff285e8c0f06a75a1fa250211bbfb2dc0a037eeef39f1a9e427" \
+    # RELATIVE_PATH="curl-{{version}}" \
+    #   install-from-source \
+    #     --disable-manual \
+    #     --disable-debug \
+    #     --enable-optimize \
+    #     --disable-static \
+    #     --disable-ldap \
+    #     --disable-ldaps \
+    #     --disable-rtsp \
+    #     --enable-proxy \
+    #     --disable-dependency-tracking \
+    #     --enable-ipv6 \
+    #     --without-libidn \
+    #     --without-gnutls \
+    #     --without-librtmp \
+    #     --without-libssh2 \
+    #     --with-ssl="${DD_PREFIX_PATH}"
+    # # Remove the binary installed so that we consistenly use the same original `curl` binary
+    # rm "${DD_PREFIX_PATH}/bin/curl"
+
+    # # Dependencies needed to build librdkafka (and thus, confluent-kafka) with kerberos support
+    # # Note that we don't ship these but rely on the Agent providing a working cyrus-sasl installation
+    # # with kerberos support, therefore we only need to watch out for the version of cyrus-sasl being
+    # # compatible with that in the Agent, the rest shouldn't matter much
+    # DOWNLOAD_URL="https://github.com/LMDB/lmdb/archive/LMDB_{{version}}.tar.gz" \
+    # VERSION="0.9.29" \
+    # SHA256="22054926b426c66d8f2bc22071365df6e35f3aacf19ad943bc6167d4cae3bebb" \
+    # RELATIVE_PATH="lmdb-LMDB_{{version}}/libraries/liblmdb" \
+    # CONFIGURE_SCRIPT="true" \
+    #   install-from-source
+    # DOWNLOAD_URL="https://github.com/cyrusimap/cyrus-sasl/releases/download/cyrus-sasl-{{version}}/cyrus-sasl-{{version}}.tar.gz" \
+    # VERSION="2.1.28" \
+    # SHA256="7ccfc6abd01ed67c1a0924b353e526f1b766b21f42d4562ee635a8ebfc5bb38c" \
+    # RELATIVE_PATH="cyrus-sasl-{{version}}" \
+    #   install-from-source --with-dblib=lmdb --enable-gssapi="${DD_PREFIX_PATH}" --disable-macos-framework
+
+    # # Cache everything under prefix
+    # if [[ -n ${DD_PREFIX_CACHE:-} ]]; then
+    #     cp -r "${DD_PREFIX_PATH}" "${DD_PREFIX_CACHE}"
+    # fi
 fi
 
 export DD_BUILD_COMMAND="bash $(pwd)/extra_build.sh"

diff --git a/.github/workflows/build-deps-macos.yml b/.github/workflows/build-deps-macos.yml
@@ -0,0 +1,39 @@
+# Workflow for dev / testing purposes only, to be migrated to a common workflow alongside
+# the rest of the deps build
+
+name: Build dependencies for mac OS
+
+on:
+  pull_request
+
+jobs:
+  build-deps-macos:
+    name: "Build wheels for integration dependencies for mac OS"
+    runs-on: macos-12
+    steps:
+      - name: "Set up environment"
+        run: |
+          # We remove everything that comes pre-installed via brew to avoid depending or shipping stuff that
+          # comes in the runner through brew to better control what might get shipped in the wheels via `delocate`
+          #brew remove --force --ignore-dependencies $(brew list --formula)
+          brew install coreutils
+      - name: "Install Python(s)"
+        env:
+          # Despite the name, this is built for the macOS 11 SDK on arm64 and 10.9+ on intel
+          PYTHON3_DOWNLOAD_URL: "https://www.python.org/ftp/python/3.11.5/python-3.11.5-macos11.pkg"
+          PYTHON2_DOWNLOAD_URL: "https://www.python.org/ftp/python/2.7.18/python-2.7.18-macosx10.9.pkg"
+        run: |
+          curl "$PYTHON3_DOWNLOAD_URL" -o python3.pkg
+          sudo installer -pkg python3.pkg -target /
+
+          curl "$PYTHON2_DOWNLOAD_URL" -o python2.pkg
+          sudo installer -pkg python2.pkg -target /
+
+      - uses: actions/checkout@v4
+      - name: "Run the build"
+        env:
+          DD_PYTHON3: "/Library/Frameworks/Python.framework/Versions/3.11/bin/python3"
+          DD_PYTHON2: "/Library/Frameworks/Python.framework/Versions/2.7/bin/python"
+        run: |
+          ${DD_PYTHON3} -m pip install packaging
+          ${DD_PYTHON3} .builders/build.py --python 3 out_py3