build-deps.yml

name: Build dependencies

on:
  workflow_dispatch:
  pull_request:
    branches:
    - master
    - 7.*.*
    paths:
    - .github/workflows/build-deps.yml
    - .builders/**
    - agent_requirements.in
  push:
    branches:
    - master
    - 7.*.*
    paths:
    - .github/workflows/build-deps.yml
    - .builders/**
    - agent_requirements.in

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' && true || false }}

defaults:
  run:
    shell: bash

env:
  PYTHONUNBUFFERED: "1"
  PYTHON_VERSION: "3.11"
  DIRECT_DEPENDENCY_FILE: agent_requirements.in
  # https://reproducible-builds.org/specs/source-date-epoch/
  SOURCE_DATE_EPOCH: "1580601600"

jobs:
  test:
    name: Run tests
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Set up Python ${{ env.PYTHON_VERSION }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}
      - name: Install dependencies
        run: |
          pip install -r .builders/deps/host_dependencies.txt
          pip install -r .builders/test_dependencies.txt
      - name: Run tests
        run: |
          cd .builders
          pytest -vvv

  build:
    name: Target ${{ matrix.job.image }} on ${{ matrix.job.os }}
    runs-on: ${{ matrix.job.os }}
    strategy:
      fail-fast: false
      matrix:
        job:
        - os: arm-4core-linux
          image: linux-aarch64
        - os: ubuntu-22.04
          image: linux-x86_64
        - os: windows-2022
          image: windows-x86_64

    permissions:
      packages: write

    env:
      OUT_DIR: output/${{ matrix.job.image }}
      BUILDER_IMAGE: ghcr.io/datadog/agent-int-builder:${{ matrix.job.image }}
      DOCKER: docker

    steps:
    - name: Checkout code
      if: github.event_name != 'pull_request'
      uses: actions/checkout@v4
      with:
        fetch-depth: 2

    # On pull requests, ensure that changed files are determined before checking out the code so
    # that we use the GitHub API, otherwise we would have to fetch the entire history (depth: 0)
    - name: Get changed files
      id: changed-files
      uses: tj-actions/changed-files@v42
      with:
        files_yaml: |-
          builders:
          - .builders/**
          dependencies:
          - ${{ env.DIRECT_DEPENDENCY_FILE }}

    - name: Checkout code
      if: github.event_name == 'pull_request'
      uses: actions/checkout@v4

    - name: Set up Python ${{ env.PYTHON_VERSION }}
      if: matrix.job.image != 'linux-aarch64'
      uses: actions/setup-python@v5
      with:
        python-version: ${{ env.PYTHON_VERSION }}

    - name: Set up Python (with miniconda) and other aarch64 requirements
      if: matrix.job.image == 'linux-aarch64'
      run: |
        mkdir -p ~/miniconda3
        wget https://repo.anaconda.com/miniconda/Miniconda3-py311_24.1.2-0-Linux-aarch64.sh -O ~/miniconda3/miniconda.sh
        bash ~/miniconda3/miniconda.sh -b -u -p ~/miniconda3
        rm -rf ~/miniconda3/miniconda.sh
        ~/miniconda3/bin/conda init bash

        # jq
        wget https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-arm64 -O ~/miniconda3/bin/jq
        chmod +x ~/miniconda3/bin/jq

        echo "PATH=~/miniconda3/bin/:${PATH}" >> "$GITHUB_ENV"
        echo DOCKER="sudo docker" >> "$GITHUB_ENV"

    - name: Install management dependencies
      run: |
        pip install -r .builders/deps/host_dependencies.txt

    - name: Install docker and log in (arm64)
      if: matrix.job.image == 'linux-aarch64'
      run: |
        curl -fsSL https://get.docker.com -o get-docker.sh
        sudo sh get-docker.sh

        # Logging in with sudo is necessary to get authorized with the registry when running docker under sudo
        echo ${{ secrets.GITHUB_TOKEN }} | sudo docker login --username ${{ github.actor }} --password-stdin ghcr.io

    - name: Log in to GitHub Packages
      if: matrix.job.image != 'linux-aarch64'
      uses: docker/login-action@v3
      with:
        registry: ghcr.io
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}

    - name: Build image and wheels (arm64)
      if: steps.changed-files.outputs.builders_any_changed == 'true' && matrix.job.image == 'linux-aarch64'
      run: |-
        sudo /home/runner/miniconda3/bin/python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3
        sudo /home/runner/miniconda3/bin/python .builders/build.py ${{ matrix.job.image }} --python 2 ${{ env.OUT_DIR }}/py2
        # Give ownership of the output back to the user
        sudo chown ${USER} ${{ env.OUT_DIR }}

    - name: Pull image and build wheels (arm64)
      if: steps.changed-files.outputs.builders_any_changed != 'true' && matrix.job.image == 'linux-aarch64'
      run: |-
        digest=$(jq -r '.["${{ matrix.job.image }}"]' .deps/image_digests.json)
        sudo /home/runner/miniconda3/bin/python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3 --digest $digest
        sudo /home/runner/miniconda3/bin/python .builders/build.py ${{ matrix.job.image }} --python 2 ${{ env.OUT_DIR }}/py2 --digest $digest

    - name: Build image and wheels
      if: steps.changed-files.outputs.builders_any_changed == 'true' && matrix.job.image != 'linux-aarch64'
      run: |-
        python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3
        python .builders/build.py ${{ matrix.job.image }} --python 2 ${{ env.OUT_DIR }}/py2

    - name: Pull image and build wheels
      if: steps.changed-files.outputs.builders_any_changed != 'true' && matrix.job.image != 'linux-aarch64'
      run: |-
        digest=$(jq -r '.["${{ matrix.job.image }}"]' .deps/image_digests.json)
        python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3 --digest $digest
        python .builders/build.py ${{ matrix.job.image }} --python 2 ${{ env.OUT_DIR }}/py2 --digest $digest

    - name: Change permissions
      if: matrix.job.image == 'linux-aarch64'
      run: |
        sudo chmod 777 ${{ env.OUT_DIR }}

    - name: Publish image
      if: github.event_name == 'push' && steps.changed-files.outputs.builders_any_changed == 'true'
      run: ${DOCKER} push ${{ env.BUILDER_IMAGE }}

    - name: Save new image digest
      if: github.event_name == 'push' && steps.changed-files.outputs.builders_any_changed == 'true'
      run: >-
        ${DOCKER} inspect --format "{{index .RepoDigests 0}}" ${{ env.BUILDER_IMAGE }}
        | cut -d '@' -f 2
        > ${{ env.OUT_DIR }}/image_digest

    - name: Persist current image digest
      if: github.event_name == 'push' && steps.changed-files.outputs.builders_any_changed != 'true'
      run: >-
        jq -r '.["${{ matrix.job.image }}"]' .deps/image_digests.json
        > ${{ env.OUT_DIR }}/image_digest

    - name: Upload artifacts
      uses: actions/upload-artifact@v4
      with:
        name: target-${{ matrix.job.image }}
        path: output

  build-macos:
    name: Target macOS
    runs-on: macos-12

    env:
      TARGET_NAME: macos-x86_64
      OUT_DIR: output/macos-x86_64
      DD_PYTHON3: "/Library/Frameworks/Python.framework/Versions/3.11/bin/python"
      DD_PYTHON2: "/Library/Frameworks/Python.framework/Versions/2.7/bin/python"

    steps:
    - name: Set up environment
      run: |-
        # We remove everything that comes pre-installed via Homebrew to avoid depending on or shipping stuff that
        # comes in the runner through Homebrew to better control what might get shipped in the wheels via `delocate`
        brew remove --force --ignore-dependencies $(brew list --formula)
        brew install coreutils

    - name: Set up Python
      env:
        # Despite the name, this is built for the macOS 11 SDK on arm64 and 10.9+ on intel
        PYTHON3_DOWNLOAD_URL: "https://www.python.org/ftp/python/3.11.5/python-3.11.5-macos11.pkg"
        PYTHON2_DOWNLOAD_URL: "https://www.python.org/ftp/python/2.7.18/python-2.7.18-macosx10.9.pkg"
      run: |-
        curl "$PYTHON3_DOWNLOAD_URL" -o python3.pkg
        sudo installer -pkg python3.pkg -target /

        curl "$PYTHON2_DOWNLOAD_URL" -o python2.pkg
        sudo installer -pkg python2.pkg -target /

    - name: Checkout code
      uses: actions/checkout@v4

    - name: Install management dependencies
      run: |
        ${DD_PYTHON3} -m pip install -r .builders/deps/host_dependencies.txt
        ${DD_PYTHON3} -m pip install --no-warn-script-location -r ".builders/images/runner_dependencies.txt"

    - name: Cache builder root
      id: cache-builder-root
      uses: actions/cache@v4
      with:
        path: |
          ~/builder_root
        key: macos-deps-builder-root-cache-${{ hashFiles('./.builders/images/macos/*', './.builders/images/*', './.builders/deps/*', './.builders/build.py', './.github/workflows/build-deps.yml') }}

    - name: Run the build
      env:
        # This sets the minimum macOS version compatible for all built artifacts
        MACOSX_DEPLOYMENT_TARGET: "10.12"
        CACHE_HIT: ${{ steps.cache-builder-root.outputs.cache-hit }}
      run: |-
        # If we hit the cache, we can skip the builder setup
        if [[ ${CACHE_HIT} == "true" ]]; then
          ${DD_PYTHON3} .builders/build.py ${{ env.TARGET_NAME }} --builder-root ~/builder_root --python 3 ${{ env.OUT_DIR }}/py3 --skip-setup
        else
          mkdir ~/builder_root
          ${DD_PYTHON3} .builders/build.py ${{ env.TARGET_NAME }} --builder-root ~/builder_root --python 3 ${{ env.OUT_DIR }}/py3
        fi
        # At this point we can always skip builder setup since it's equivalent for python versions
        ${DD_PYTHON3} .builders/build.py ${{ env.TARGET_NAME }} --builder-root ~/builder_root --python 2 ${{ env.OUT_DIR }}/py2 --skip-setup

    - name: Upload artifacts
      uses: actions/upload-artifact@v4
      with:
        name: target-macos
        path: output

  publish:
    name: Publish artifacts
    if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && (github.ref == github.event.repository.default_branch || startsWith(github.ref, '7.')))
    needs:
    - build
    - build-macos
    runs-on: ubuntu-latest

    permissions:
      contents: read
      id-token: write

    steps:
    - name: Checkout code
      uses: actions/checkout@v4
      with:
        ref: "${{ github.head_ref }}"

    - name: Set up Python ${{ env.PYTHON_VERSION }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ env.PYTHON_VERSION }}

    - name: Install management dependencies
      run: pip install -r .builders/deps/host_dependencies.txt

    - name: Download artifacts
      uses: actions/download-artifact@v4
      with:
        path: targets
        pattern: target-*
        merge-multiple: true

    - name: Get credentials
      id: auth
      uses: google-github-actions/auth@v2
      with:
        project_id: datadog-agent-int-build
        workload_identity_provider: projects/574011472402/locations/global/workloadIdentityPools/github/providers/integrations-core

    - name: Upload wheels
      run: python .builders/upload.py targets

    - name: Lock files
      run: python .builders/lock.py targets

    - name: Clean up repository
      run: |-
        rm ${{ steps.auth.outputs.credentials_file_path }}
        rm -rf targets

    - name: Create token
      uses: actions/create-github-app-token@v1
      id: token-generator
      with:
        app-id: ${{ vars.DD_GITHUB_TOKEN_GENERATOR_APP_ID }}
        private-key: ${{ secrets.DD_GITHUB_TOKEN_GENERATOR_PRIVATE_KEY }}
        repositories: integrations-core

    - name: Create pull request
      uses: peter-evans/create-pull-request@v5
      with:
        token: ${{ steps.token-generator.outputs.token }}
        title: Update dependency resolution
        commit-message: Update dependency resolution
        branch: bot/update-dependency-resolution
        branch-suffix: timestamp
        delete-branch: true
        labels: bot,qa/skip-qa
        body: |-
          ### Motivation

          Direct dependencies were updated in ${{ github.sha }}.

          ### Additional Notes

          This PR was automatically generated by the following workflow:
          ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}