Merge branch 'master' into sarah/add-octopus-integration

.builders/deps/build_dependencies.txt

@@ -1,13 +1,9 @@
-hatchling==1.21.1; python_version > '3.0'
-hatchling==0.25.1; python_version < '3.0'
-setuptools==66.1.1; python_version > '3.0'
-setuptools==40.9.0; python_version < '3.0'
-wheel==0.38.4; python_version > '3.0'
-wheel==0.37.1; python_version < '3.0'
-setuptools-scm; python_version > '3.0'
-setuptools-scm==5.0.2; python_version < '3.0'
-setuptools-rust>=1.7.0; python_version > '3.0'
-maturin; python_version > '3.0'
+hatchling==1.21.1
+setuptools==75.6.0
+wheel==0.38.4
+setuptools-scm
+setuptools-rust>=1.7.0
+maturin
 cffi>=1.12
-cython<3.0.0
-tomli>=2.0.1; python_version > '3.0'
+cython==3.0.11
+tomli>=2.0.1

.builders/images/linux-aarch64/Dockerfile

@@ -15,24 +15,6 @@ ENV CXXFLAGS="${CFLAGS}"
 # --strip-debug reduces binary sizes and improves reproducibility
 ENV LDFLAGS="-Wl,-rpath,'\$\$ORIGIN' -Wl,--strip-debug"
 
-# For Python 2, we get openssl(1) via yum just to get the necessary headers,
-# and we remove the package after compilation to prevent it from interfering
-# with the use of openssl3 everywhere else
-ENV PYTHON2_VERSION=2.7.18
-RUN yum install -y openssl-devel && \
- DOWNLOAD_URL="https://python.org/ftp/python/${PYTHON2_VERSION}/Python-${PYTHON2_VERSION}.tgz" \
- VERSION="${PYTHON2_VERSION}" \
- SHA256="da3080e3b488f648a3d7a4560ddee895284c3380b11d6de75edb986526b9a814" \
- RELATIVE_PATH=Python-{{version}} \
- INSTALL_COMMAND="make altinstall" \
- bash install-from-source.sh \
- --prefix=/opt/python/${PYTHON2_VERSION} --with-ensurepip=yes --enable-ipv6 --enable-unicode=ucs4 \
- && yum remove -y openssl-devel
-
-# Set up virtual environment for Python 2
-RUN /opt/python/${PYTHON2_VERSION}/bin/python2.7 -m pip install --no-warn-script-location virtualenv \
- && /opt/python/${PYTHON2_VERSION}/bin/python2.7 -m virtualenv /py2
-
 # openssl
 RUN yum install -y perl-IPC-Cmd perl-CPANPLUS && \
  cpanp -i List::Util 1.66 && \

.builders/images/linux-aarch64/build_script.sh

@@ -21,7 +21,7 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then
     LDFLAGS="${LDFLAGS} -L/usr/local/lib -lkrb5 -lgssapi_krb5 -llmdb" \
     DOWNLOAD_URL="https://github.com/confluentinc/librdkafka/archive/refs/tags/v{{version}}.tar.gz" \
         VERSION="${kafka_version}" \
-        SHA256="3dc62de731fd516dfb1032861d9a580d4d0b5b0856beb0f185d06df8e6c26259" \
+        SHA256="0ddf205ad8d36af0bc72a2fec20639ea02e1d583e353163bf7f4683d949e901b" \
         RELATIVE_PATH="librdkafka-{{version}}" \
         bash install-from-source.sh --enable-sasl --enable-curl
     always_build+=("confluent-kafka")

.builders/images/linux-x86_64/Dockerfile

@@ -15,23 +15,6 @@ ENV CXXFLAGS="${CFLAGS}"
 # --strip-debug reduces binary sizes and improves reproducibility
 ENV LDFLAGS="-Wl,-rpath,'\$\$ORIGIN' -Wl,--strip-debug"
 
-# For Python 2, we get openssl(1) via yum just to get the necessary headers,
-# and we remove the package after compilation to prevent it from interfering
-# with the use of openssl3 everywhere else
-ENV PYTHON2_VERSION=2.7.18
-RUN yum install -y openssl-devel && \
- DOWNLOAD_URL="https://python.org/ftp/python/${PYTHON2_VERSION}/Python-${PYTHON2_VERSION}.tgz" \
- VERSION="${PYTHON2_VERSION}" \
- SHA256="da3080e3b488f648a3d7a4560ddee895284c3380b11d6de75edb986526b9a814" \
- RELATIVE_PATH=Python-{{version}} \
- bash install-from-source.sh \
- --prefix=/opt/python/${PYTHON2_VERSION} --with-ensurepip=yes --enable-ipv6 --enable-unicode=ucs4 \
- && yum remove -y openssl-devel
-
-# Set up virtual environment for Python 2
-RUN /opt/python/${PYTHON2_VERSION}/bin/python -m pip install --no-warn-script-location virtualenv \
- && /opt/python/${PYTHON2_VERSION}/bin/python -m virtualenv /py2
-
 # openssl
 RUN yum install -y perl-IPC-Cmd perl-CPANPLUS && \
  cpanp -i List::Util 1.66 && \

.builders/images/linux-x86_64/build_script.sh

@@ -18,7 +18,7 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then
     LDFLAGS="${LDFLAGS} -L/usr/local/lib -lkrb5 -lgssapi_krb5 -llmdb" \
     DOWNLOAD_URL="https://github.com/confluentinc/librdkafka/archive/refs/tags/v{{version}}.tar.gz" \
         VERSION="${kafka_version}" \
-        SHA256="3dc62de731fd516dfb1032861d9a580d4d0b5b0856beb0f185d06df8e6c26259" \
+        SHA256="0ddf205ad8d36af0bc72a2fec20639ea02e1d583e353163bf7f4683d949e901b" \
         RELATIVE_PATH="librdkafka-{{version}}" \
         bash install-from-source.sh --enable-sasl --enable-curl
     always_build+=("confluent-kafka")

.builders/images/macos-x86_64/extra_build.sh

@@ -13,7 +13,7 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then
     LDFLAGS="${LDFLAGS} -L${DD_PREFIX_PATH}/lib -lgssapi_krb5 -llmdb" \
     DOWNLOAD_URL="https://github.com/confluentinc/librdkafka/archive/refs/tags/v{{version}}.tar.gz" \
       VERSION="${kafka_version}" \
-      SHA256="3dc62de731fd516dfb1032861d9a580d4d0b5b0856beb0f185d06df8e6c26259" \
+      SHA256="0ddf205ad8d36af0bc72a2fec20639ea02e1d583e353163bf7f4683d949e901b" \
       RELATIVE_PATH="librdkafka-{{version}}" \
       bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" --enable-sasl --enable-curl
 

.builders/images/windows-x86_64/Dockerfile

@@ -35,6 +35,15 @@ RUN curl -SL --output PowerShell-%POWERSHELL_VERSION%-win-x64.msi https://github
 COPY helpers.ps1 C:\helpers.ps1
 SHELL ["pwsh", "-Command", ". C:\\helpers.ps1;"]
 
+# Enable long paths
+# https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation?tabs=powershell#registry-setting-to-enable-long-paths
+RUN New-ItemProperty -Path "HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem" `
+ -Name "LongPathsEnabled" -Value 1 -PropertyType DWORD -Force
+# Reduce the chance of hitting path limits (the MSVC compiler cl.exe doesn't seem to respect that optioin)
+# This variable is honored by pip
+ENV TMP="C:\tmp" `
+    TEMP="C:\tmp"
+
 # Install 7-Zip ZS
 ENV 7ZIP_VERSION="22.01" `
     7ZIP_ZS_VERSION="1.5.5-R3"
@@ -85,24 +94,6 @@ RUN Get-RemoteFile `
     & 'C:\Program Files\Python312\python.exe' -m virtualenv 'C:\py3'; `
     Add-ToPath -Append 'C:\Program Files\Python312'
 
-# Install Python 2
-ENV PYTHON_VERSION="2.7.18"
-RUN Get-RemoteFile `
-      -Uri https://www.python.org/ftp/python/$Env:PYTHON_VERSION/python-$Env:PYTHON_VERSION.amd64.msi `
-      -Path python-$Env:PYTHON_VERSION.amd64.msi `
-      -Hash 'b74a3afa1e0bf2a6fc566a7b70d15c9bfabba3756fb077797d16fffa27800c05'; `
-    Start-Process -Wait -FilePath msiexec -ArgumentList '/i', python-$Env:PYTHON_VERSION.amd64.msi, '/quiet', '/norestart', 'InstallAllUsers=1'; `
-    Remove-Item python-$Env:PYTHON_VERSION.amd64.msi; `
-    & 'C:\Python27\python.exe' -m pip install --no-warn-script-location --upgrade pip; `
-    & 'C:\Python27\python.exe' -m pip install --no-python-version-warning --no-warn-script-location virtualenv; `
-    & 'C:\Python27\python.exe' -m virtualenv 'C:\py2'
-RUN Get-RemoteFile `
-      -Uri https://s3.amazonaws.com/dd-agent-omnibus/VCForPython27.msi `
-      -Path VCForPython27.msi `
-      -Hash '070474db76a2e625513a5835df4595df9324d820f9cc97eab2a596dcbc2f5cbf'; `
-    Start-Process -Wait -FilePath msiexec -ArgumentList '/i', VCForPython27.msi, '/quiet'; `
-    Remove-Item VCForPython27.msi
-
 # Install IBM MQ
 ENV IBM_MQ_VERSION="9.2.4.0"
 RUN Get-RemoteFile `
@@ -113,10 +104,38 @@ RUN Get-RemoteFile `
     Remove-Item $Env:IBM_MQ_VERSION-IBM-MQC-Redist-Win64.zip; `
     setx /M MQ_FILE_PATH 'C:\ibm_mq'
 
+# Perl
+ENV PERL_VERSION="5.40.0.1"
+RUN Get-RemoteFile `
+      -Uri https://github.com/StrawberryPerl/Perl-Dist-Strawberry/releases/download/SP_54001_64bit_UCRT/strawberry-perl-$Env:PERL_VERSION-64bit-portable.zip `
+      -Path "strawberry-perl-$Env:PERL_VERSION-64bit.zip" `
+    -Hash '754f3e2a8e473dc68d1540c7802fb166a025f35ef18960c4564a31f8b5933907' && `
+    7z x "strawberry-perl-$Env:PERL_VERSION-64bit.zip" -o"C:\perl" && `
+    Add-ToPath -Append "C:\perl\perl\bin" && `
+    Remove-Item "strawberry-perl-$Env:PERL_VERSION-64bit.zip"
+
+ENV OPENSSL_VERSION="3.3.2"
+
+ENV CURL_VERSION="8.11.1"
+
 # Set up runner
 COPY runner_dependencies.txt C:\runner_dependencies.txt
 RUN python -m pip install --no-warn-script-location -r C:\runner_dependencies.txt
 
+COPY build_script.ps1 C:\build_script.ps1
+COPY update_librdkafka_manifest.py C:\update_librdkafka_manifest.py
+ENV DD_BUILD_COMMAND="pwsh C:\build_script.ps1"
+
+# Python packages that we want to build regardless of whether prebuilt versions exist on PyPI
+ENV PIP_NO_BINARY="confluent_kafka"
+# Where to find native dependencies when building extensions and for wheel repairing
+RUN New-Item -Path "C:\include" -ItemType Directory
+RUN New-Item -Path "C:\lib" -ItemType Directory
+RUN New-Item -Path "C:\bin" -ItemType Directory
+ENV INCLUDE="C:\include"
+ENV LIB="C:\lib"
+RUN Add-ToPath -Append "C:\bin"
+
 # Restore the default Windows shell for correct batch processing.
 SHELL ["cmd", "/S", "/C"]
 

.builders/images/windows-x86_64/build_script.ps1

@@ -0,0 +1,68 @@
+$ErrorActionPreference = 'Stop'
+$PSNativeCommandUseErrorActionPreference = $true
+
+. C:\helpers.ps1
+
+# The librdkafka version needs to stay in sync with the confluent-kafka version,
+# thus we extract the version from the requirements file
+$kafka_version = Get-Content 'C:\mnt\requirements.in' | perl -nE 'say/^\D*(\d+\.\d+\.\d+)\D*$/ if /confluent-kafka==/'
+Write-Host "Will build librdkafka $kafka_version"
+
+# Download and unpack the source
+Get-RemoteFile `
+  -Uri "https://github.com/confluentinc/librdkafka/archive/refs/tags/v${kafka_version}.tar.gz" `
+  -Path "librdkafka-${kafka_version}.tar.gz" `
+  -Hash '0ddf205ad8d36af0bc72a2fec20639ea02e1d583e353163bf7f4683d949e901b'
+7z x "librdkafka-${kafka_version}.tar.gz" -o"C:\"
+7z x "C:\librdkafka-${kafka_version}.tar" -o"C:\librdkafka"
+Remove-Item "librdkafka-${kafka_version}.tar.gz"
+
+# Build librdkafka
+# Based on this job from upstream:
+# https://github.com/confluentinc/librdkafka/blob/cb8c19c43011b66c4b08b25e5150455a247e1ff3/.semaphore/semaphore.yml#L265
+# Install vcpkg
+$triplet = "x64-windows"
+$vcpkg_dir = "C:\vcpkg"
+$librdkafka_dir = "C:\librdkafka\librdkafka-${kafka_version}"
+$desired_tag = "2024.12.16"
+
+# Clone and configure vcpkg
+if (-Not (Test-Path -Path "$vcpkg_dir\.git")) {
+    git clone https://github.com/Microsoft/vcpkg.git $vcpkg_dir
+}
+
+Set-Location $vcpkg_dir
+git checkout $desired_tag
+
+Write-Host "Bootstrapping vcpkg..."
+.\bootstrap-vcpkg.bat
+
+# Get deps
+Set-Location "$librdkafka_dir"
+# Patch the the vcpkg manifest to to override the OpenSSL version and CURL version
+python C:\update_librdkafka_manifest.py vcpkg.json --set-version openssl:${Env:OPENSSL_VERSION} --set-version curl:${Env:CURL_VERSION}
+
+C:\vcpkg\vcpkg integrate install
+C:\vcpkg\vcpkg --feature-flags=versions install --triplet $triplet
+# Build
+& .\win32\msbuild.ps1 -platform x64
+
+# Copy outputs to where they can be found
+# This is partially inspired by
+# https://github.com/confluentinc/librdkafka/blob/cb8c19c43011b66c4b08b25e5150455a247e1ff3/win32/package-zip.ps1
+$toolset = "v142"
+$platform = "x64"
+$config = "Release"
+$srcdir = "win32\outdir\${toolset}\${platform}\$config"
+$bindir = "C:\bin"
+$libdir = "C:\lib"
+$includedir = "C:\include"
+
+Copy-Item "${srcdir}\librdkafka.dll","${srcdir}\librdkafkacpp.dll",
+"${srcdir}\libcrypto-3-x64.dll","${srcdir}\libssl-3-x64.dll",
+"${srcdir}\zlib1.dll","${srcdir}\zstd.dll","${srcdir}\libcurl.dll" -Destination $bindir
+Copy-Item "${srcdir}\librdkafka.lib","${srcdir}\librdkafkacpp.lib" -Destination $libdir
+
+New-Item -Path $includedir\librdkafka -ItemType Directory
+Copy-Item -Path ".\src\*" -Filter *.h -Destination $includedir\librdkafka
+

.builders/images/windows-x86_64/update_librdkafka_manifest.py

@@ -0,0 +1,25 @@
+import json
+from argparse import ArgumentParser
+
+
+def main(manifest_file, versions):
+    with open(manifest_file) as f:
+        manifest = json.load(f)
+
+    for dep, version in versions.items():
+        manifest.setdefault("overrides", []).append({
+            "name": dep,
+            "version": version,
+        })
+
+    with open(manifest_file, 'w') as f:
+        json.dump(manifest, f)
+
+
+if __name__ == '__main__':
+    ap = ArgumentParser()
+    ap.add_argument("file")
+    ap.add_argument("--set-version", action="append", required=True)
+
+    args = ap.parse_args()
+    main(args.file, dict(spec.split(':') for spec in args.set_version))

.builders/scripts/repair_wheels.py

@@ -6,10 +6,12 @@
 import shutil
 import sys
 import time
+from fnmatch import fnmatch
 from functools import cache
 from hashlib import sha256
 from pathlib import Path
 from typing import Iterator, NamedTuple
+from zipfile import ZipFile
 
 import urllib3
 from utils import extract_metadata, normalize_project_name
@@ -64,6 +66,14 @@ def wheel_was_built(wheel: Path) -> bool:
     return file_hash != wheel_hashes[wheel.name]
 
 
+def find_patterns_in_wheel(wheel: Path, patterns: list[str]) -> list[str]:
+    """Returns all found files inside `wheel` that match the given glob-style pattern"""
+    with ZipFile(wheel) as zf:
+        names = zf.namelist()
+
+    return [name for name in names for pat in patterns if fnmatch(name, pat)]
+
+
 class WheelName(NamedTuple):
     """Helper class to manipulate wheel names."""
     # Note: this implementation ignores build tags (it drops them on parsing)
@@ -98,6 +108,12 @@ def repair_linux(source_dir: str, built_dir: str, external_dir: str) -> None:
         'libmqic_r.so',
     })
 
+    external_invalid_file_patterns = [
+        # We don't accept OpenSSL in external wheels
+        '*.libs/libssl*.so.3',
+        '*.libs/libcrypto*.so.3',
+    ]
+
     # Hardcoded policy to the minimum we need to currently support
     policies = WheelPolicies()
     policy = policies.get_policy_by_name(os.environ['MANYLINUX_POLICY'])
@@ -109,8 +125,19 @@ def repair_linux(source_dir: str, built_dir: str, external_dir: str) -> None:
 
     for wheel in iter_wheels(source_dir):
         print(f'--> {wheel.name}')
+
         if not wheel_was_built(wheel):
             print('Using existing wheel')
+
+            unacceptable_files = find_patterns_in_wheel(wheel, external_invalid_file_patterns)
+            if unacceptable_files:
+                print(
+                    f"Found copies of unacceptable files in external wheel '{wheel.name}'",
+                    f'(matching {external_invalid_file_patterns}): ',
+                    unacceptable_files,
+                )
+                sys.exit(1)
+
             shutil.move(wheel, external_dir)
             continue
 
@@ -140,11 +167,27 @@ def repair_windows(source_dir: str, built_dir: str, external_dir: str) -> None:
 
     exclusions = ['mqic.dll']
 
+    external_invalid_file_patterns = [
+        # We don't accept OpenSSL in external wheels
+        '*.libs/libssl-3*.dll',
+        '*.libs/libcrypto-3*.dll',
+    ]
+
     for wheel in iter_wheels(source_dir):
         print(f'--> {wheel.name}')
 
         if not wheel_was_built(wheel):
             print('Using existing wheel')
+
+            unacceptable_files = find_patterns_in_wheel(wheel, external_invalid_file_patterns)
+            if unacceptable_files:
+                print(
+                    f"Found copies of unacceptable files in external wheel '{wheel.name}'",
+                    f'(matching {external_invalid_file_patterns}): ',
+                    unacceptable_files,
+                )
+                sys.exit(1)
+
             shutil.move(wheel, external_dir)
             continue
 
@@ -206,6 +249,7 @@ def copy_filt_func(libname):
         print(f'--> {wheel.name}')
         if not wheel_was_built(wheel):
             print('Using existing wheel')
+
             shutil.move(wheel, external_dir)
             continue