Skip to content

Bump Python 3 to 3.12 #791

Bump Python 3 to 3.12

Bump Python 3 to 3.12 #791

Workflow file for this run

name: Build dependencies
on:
workflow_dispatch:
pull_request:
branches:
- master
- 7.*.*
paths:
- .github/workflows/build-deps.yml
- .builders/**
- agent_requirements.in
push:
branches:
- master
- 7.*.*
paths:
- .builders/**
- agent_requirements.in
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' && true || false }}
defaults:
run:
shell: bash
env:
PYTHONUNBUFFERED: "1"
PYTHON_VERSION: "3.12"
DIRECT_DEPENDENCY_FILE: agent_requirements.in
# https://reproducible-builds.org/specs/source-date-epoch/
SOURCE_DATE_EPOCH: "1580601600"
jobs:
test:
name: Run tests
runs-on: ubuntu-22.04
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set up Python ${{ env.PYTHON_VERSION }}
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
- name: Install dependencies
run: |
pip install -r .builders/deps/host_dependencies.txt
pip install -r .builders/test_dependencies.txt
- name: Run tests
run: |
cd .builders
pytest -vvv
build:
name: Target ${{ matrix.job.image }} on ${{ matrix.job.os }}
runs-on: ${{ matrix.job.os }}
strategy:
fail-fast: false
matrix:
job:
- os: arm-4core-linux
image: linux-aarch64
- os: ubuntu-22.04
image: linux-x86_64
- os: windows-2022
image: windows-x86_64
permissions:
packages: write
env:
OUT_DIR: output/${{ matrix.job.image }}
BUILDER_IMAGE: ghcr.io/datadog/agent-int-builder:${{ matrix.job.image }}
DOCKER: docker
steps:
- name: Checkout code
if: github.event_name != 'pull_request'
uses: actions/checkout@v4
with:
fetch-depth: 2
# On pull requests, ensure that changed files are determined before checking out the code so
# that we use the GitHub API, otherwise we would have to fetch the entire history (depth: 0)
- name: Get changed files
id: changed-files
uses: tj-actions/changed-files@v42
with:
files_yaml: |-
builders:
- .builders/**
dependencies:
- ${{ env.DIRECT_DEPENDENCY_FILE }}
- name: Checkout code
if: github.event_name == 'pull_request'
uses: actions/checkout@v4
- name: Set up Python ${{ env.PYTHON_VERSION }}
if: matrix.job.image != 'linux-aarch64'
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
- name: Set up Python (with miniconda) and other aarch64 requirements
if: matrix.job.image == 'linux-aarch64'
run: |
mkdir -p ~/miniconda3
wget https://repo.anaconda.com/miniconda/Miniconda3-py312_24.5.0-0-Linux-aarch64.sh -O ~/miniconda3/miniconda.sh
bash ~/miniconda3/miniconda.sh -b -u -p ~/miniconda3
rm -rf ~/miniconda3/miniconda.sh
~/miniconda3/bin/conda init bash
# jq
wget https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-arm64 -O ~/miniconda3/bin/jq
chmod +x ~/miniconda3/bin/jq
echo "PATH=~/miniconda3/bin/:${PATH}" >> "$GITHUB_ENV"
echo DOCKER="sudo docker" >> "$GITHUB_ENV"
- name: Install management dependencies
run: |
pip install -r .builders/deps/host_dependencies.txt
- name: Install docker and log in (arm64)
if: matrix.job.image == 'linux-aarch64'
run: |
curl -fsSL https://get.docker.com -o get-docker.sh
sudo sh get-docker.sh
# Logging in with sudo is necessary to get authorized with the registry when running docker under sudo
echo ${{ secrets.GITHUB_TOKEN }} | sudo docker login --username ${{ github.actor }} --password-stdin ghcr.io
- name: Log in to GitHub Packages
if: matrix.job.image != 'linux-aarch64'
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build image and wheels (arm64)
if: steps.changed-files.outputs.builders_any_changed == 'true' && matrix.job.image == 'linux-aarch64'
run: |-
sudo /home/runner/miniconda3/bin/python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3
# Give ownership of the output back to the user
sudo chown ${USER} ${{ env.OUT_DIR }}
- name: Pull image and build wheels (arm64)
if: steps.changed-files.outputs.builders_any_changed != 'true' && matrix.job.image == 'linux-aarch64'
run: |-
digest=$(jq -r '.["${{ matrix.job.image }}"]' .deps/image_digests.json)
sudo /home/runner/miniconda3/bin/python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3 --digest $digest
- name: Build image and wheels
if: steps.changed-files.outputs.builders_any_changed == 'true' && matrix.job.image != 'linux-aarch64'
run: |-
python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3
- name: Pull image and build wheels
if: steps.changed-files.outputs.builders_any_changed != 'true' && matrix.job.image != 'linux-aarch64'
run: |-
digest=$(jq -r '.["${{ matrix.job.image }}"]' .deps/image_digests.json)
python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3 --digest $digest
- name: Change permissions
if: matrix.job.image == 'linux-aarch64'
run: |
sudo chmod 777 ${{ env.OUT_DIR }}
- name: Publish image
if: github.event_name == 'push' && steps.changed-files.outputs.builders_any_changed == 'true'
run: ${DOCKER} push ${{ env.BUILDER_IMAGE }}
- name: Save new image digest
if: github.event_name == 'push' && steps.changed-files.outputs.builders_any_changed == 'true'
run: >-
${DOCKER} inspect --format "{{index .RepoDigests 0}}" ${{ env.BUILDER_IMAGE }}
| cut -d '@' -f 2
> ${{ env.OUT_DIR }}/image_digest
- name: Persist current image digest
if: github.event_name == 'push' && steps.changed-files.outputs.builders_any_changed != 'true'
run: >-
jq -r '.["${{ matrix.job.image }}"]' .deps/image_digests.json
> ${{ env.OUT_DIR }}/image_digest
- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
name: target-${{ matrix.job.image }}
path: output
build-macos:
name: Target macOS
runs-on: macos-12
env:
TARGET_NAME: macos-x86_64
OUT_DIR: output/macos-x86_64
DD_PYTHON3: "/Library/Frameworks/Python.framework/Versions/3.12/bin/python"
steps:
- name: Set up environment
run: |-
# We remove everything that comes pre-installed via Homebrew to avoid depending on or shipping stuff that
# comes in the runner through Homebrew to better control what might get shipped in the wheels via `delocate`
brew remove --force --ignore-dependencies $(brew list --formula)
brew install coreutils
- name: Set up Python
env:
# Despite the name, this is built for the macOS 11 SDK on arm64 and 10.9+ on intel
PYTHON3_DOWNLOAD_URL: "https://www.python.org/ftp/python/3.12.6/python-3.12.6-macos11.pkg"
run: |-
curl "$PYTHON3_DOWNLOAD_URL" -o python3.pkg
sudo installer -pkg python3.pkg -target /
- name: Checkout code
uses: actions/checkout@v4
- name: Install management dependencies
run: |
${DD_PYTHON3} -m pip install -r .builders/deps/host_dependencies.txt
${DD_PYTHON3} -m pip install --no-warn-script-location -r ".builders/images/runner_dependencies.txt"
- name: Cache builder root
id: cache-builder-root
uses: actions/cache@v4
with:
path: |
~/builder_root
key: macos-deps-builder-root-cache-${{ hashFiles('./.builders/images/macos/*', './.builders/images/*', './.builders/deps/*', './.builders/build.py') }}
- name: Run the build
env:
# This sets the minimum macOS version compatible for all built artifacts
MACOSX_DEPLOYMENT_TARGET: "10.12"
CACHE_HIT: ${{ steps.cache-builder-root.outputs.cache-hit }}
run: |-
# If we hit the cache, we can skip the builder setup
if [[ ${CACHE_HIT} == "true" ]]; then
${DD_PYTHON3} .builders/build.py ${{ env.TARGET_NAME }} --builder-root ~/builder_root --python 3 ${{ env.OUT_DIR }}/py3 --skip-setup
else
mkdir ~/builder_root
${DD_PYTHON3} .builders/build.py ${{ env.TARGET_NAME }} --builder-root ~/builder_root --python 3 ${{ env.OUT_DIR }}/py3
fi
- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
name: target-macos
path: output
publish:
name: Publish artifacts
if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && (github.ref == github.event.repository.default_branch || startsWith(github.ref, '7.')))
needs:
- build
- build-macos
runs-on: ubuntu-latest
permissions:
contents: read
id-token: write
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: "${{ github.head_ref }}"
- name: Set up Python ${{ env.PYTHON_VERSION }}
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
- name: Install management dependencies
run: pip install -r .builders/deps/host_dependencies.txt
- name: Download artifacts
uses: actions/download-artifact@v4
with:
path: targets
pattern: target-*
merge-multiple: true
- name: Get credentials
id: auth
uses: google-github-actions/auth@v2
with:
project_id: datadog-agent-int-build
workload_identity_provider: projects/574011472402/locations/global/workloadIdentityPools/github/providers/integrations-core
- name: Upload wheels
run: python .builders/upload.py targets
- name: Lock files
run: python .builders/lock.py targets
- name: Clean up repository
run: |-
rm ${{ steps.auth.outputs.credentials_file_path }}
rm -rf targets
- name: Create token
uses: actions/create-github-app-token@v1
id: token-generator
with:
app-id: ${{ vars.DD_GITHUB_TOKEN_GENERATOR_APP_ID }}
private-key: ${{ secrets.DD_GITHUB_TOKEN_GENERATOR_PRIVATE_KEY }}
repositories: integrations-core
- name: Create pull request
uses: peter-evans/create-pull-request@v5
with:
token: ${{ steps.token-generator.outputs.token }}
title: Update dependency resolution
commit-message: Update dependency resolution
branch: bot/update-dependency-resolution
branch-suffix: timestamp
delete-branch: true
labels: bot,qa/skip-qa
body: |-
### Motivation
Direct dependencies were updated in ${{ github.sha }}.
### Additional Notes
This PR was automatically generated by the following workflow:
${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}