concrete-python release #530
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Concrete Python Release | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| instance_id: | |
| description: 'Instance ID' | |
| type: string | |
| instance_image_id: | |
| description: 'Instance AMI ID' | |
| type: string | |
| instance_type: | |
| description: 'Instance product type' | |
| type: string | |
| runner_name: | |
| description: 'Action runner name' | |
| type: string | |
| request_id: | |
| description: 'Slab request ID' | |
| type: string | |
| user_inputs: | |
| description: 'either "nightly" or "public" or "private" to specify the release type' | |
| required: true | |
| default: 'nightly' | |
| type: string | |
| env: | |
| DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler | |
| GLIB_VER: 2_28 | |
| RELEASE_TYPE: ${{ inputs.user_inputs }} | |
| jobs: | |
| release-checks: | |
| runs-on: ${{ github.event.inputs.runner_name }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 | |
| - name: Check python api doc is up to date | |
| run: ci/scripts/make_apidocs.sh | |
| build-linux-x86: | |
| strategy: | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] | |
| runs-on: ${{ github.event.inputs.runner_name }} | |
| steps: | |
| - name: Log instance configuration | |
| run: | | |
| echo "IDs: ${{ inputs.instance_id }}" | |
| echo "AMI: ${{ inputs.instance_image_id }}" | |
| echo "Type: ${{ inputs.instance_type }}" | |
| echo "Request ID: ${{ inputs.request_id }}" | |
| echo "User Inputs: ${{ inputs.user_inputs }}" | |
| - name: Set up GitHub environment | |
| run: | | |
| echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" | |
| - name: Checkout | |
| uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 | |
| - name: Set release version (nightly) | |
| if: ${{ env.RELEASE_TYPE == 'nightly' }} | |
| run: | | |
| NIGHTLY_VERSION=$(date +"%Y.%m.%d") | |
| NIGHTLY_VERSION_ONE_NUMBER=$(date +"%Y%m%d") | |
| LATEST_RELEASE_VERSION=`git tag -l |grep "v.*" |sort |tail -n 1 | grep -e '[0-9].*' -o` | |
| echo "__version__ = \"${LATEST_RELEASE_VERSION}-dev${NIGHTLY_VERSION_ONE_NUMBER}\"" >| frontends/concrete-python/version.txt | |
| git tag nightly-$NIGHTLY_VERSION || true | |
| git push origin nightly-$NIGHTLY_VERSION || true | |
| - name: Set release version (public) | |
| if: ${{ env.RELEASE_TYPE == 'public' }} | |
| run: echo "__version__ = \"`git describe --tags --abbrev=0 | grep -e '[0-9].*' -o`\"" >| frontends/concrete-python/version.txt | |
| - name: Expose release version from Python | |
| run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py | |
| - name: Build wheel | |
| uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 | |
| id: build-compiler-bindings | |
| with: | |
| registry: ghcr.io | |
| image: ${{ env.DOCKER_IMAGE_TEST }} | |
| username: ${{ secrets.GHCR_LOGIN }} | |
| password: ${{ secrets.GHCR_PASSWORD }} | |
| options: >- | |
| -v ${{ github.workspace }}:/concrete | |
| -v ${{ github.workspace }}/build:/build | |
| -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket | |
| -e SSH_AUTH_SOCK=/ssh.socket | |
| ${{ env.DOCKER_GPU_OPTION }} | |
| shell: bash | |
| run: | | |
| set -e | |
| rustup toolchain install nightly-2024-09-30 | |
| pip install mypy | |
| rm -rf /build/* | |
| export PYTHON=${{ format('python{0}', matrix.python-version) }} | |
| echo "Using $PYTHON" | |
| dnf -y install graphviz graphviz-devel | |
| cd /concrete/frontends/concrete-python | |
| make PYTHON=$PYTHON venv | |
| source .venv/bin/activate | |
| cd /concrete/compilers/concrete-compiler/compiler | |
| make BUILD_DIR=/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=ON Python3_EXECUTABLE=$(which python) python-bindings | |
| echo "Debug: ccache statistics (after the build):" | |
| ccache -s | |
| cd /concrete/frontends/concrete-python | |
| export COMPILER_BUILD_DIRECTORY="/build" | |
| make whl | |
| deactivate | |
| - name: Upload wheel | |
| uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 | |
| with: | |
| name: ${{ format('wheel-{0}-linux-x86', matrix.python-version) }} | |
| path: frontends/concrete-python/dist/*manylinux*.whl | |
| retention-days: 3 | |
| build-macos: | |
| strategy: | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] | |
| runs-on: ["aws-mac1-metal", "aws-mac2-metal"] | |
| runs-on: ${{ matrix.runs-on }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 | |
| - name: Install OS Dependencies | |
| run: | | |
| brew install ninja ccache | |
| - name: Setup rust toolchain for concrete-cpu | |
| uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu | |
| - name: Set release version (nightly) | |
| if: ${{ env.RELEASE_TYPE == 'nightly' }} | |
| run: | | |
| NIGHTLY_VERSION=$(date +"%Y.%m.%d") | |
| NIGHTLY_VERSION_ONE_NUMBER=$(date +"%Y%m%d") | |
| LATEST_RELEASE_VERSION=`git tag -l |grep "v.*" |sort |tail -n 1 | grep -e '[0-9].*' -o` | |
| echo "__version__ = \"${LATEST_RELEASE_VERSION}-dev${NIGHTLY_VERSION_ONE_NUMBER}\"" >| frontends/concrete-python/version.txt | |
| - name: Set release version (public) | |
| if: ${{ env.RELEASE_TYPE == 'public' }} | |
| run: echo "__version__ = \"`git describe --tags --abbrev=0 | grep -e '[0-9].*' -o`\"" >| frontends/concrete-python/version.txt | |
| - name: Expose release version from Python | |
| run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py | |
| - name: Build wheel | |
| run: | | |
| export CONCRETE_PYTHON=$(pwd)/frontends/concrete-python | |
| export CONCRETE_COMPILER=$(pwd)/compilers/concrete-compiler/compiler | |
| export COMPILER_BUILD_DIRECTORY=$CONCRETE_COMPILER/build | |
| export PYTHON=${{ format('python{0}', matrix.python-version) }} | |
| echo "Using $PYTHON" | |
| # Setup pkg-config to find OpenBLAS (scipy need it) | |
| export PKG_CONFIG_PATH="/opt/homebrew/opt/openblas/lib/pkgconfig" | |
| # Setup vitual environment | |
| rm -rf .venv | |
| $PYTHON -m venv .venv && . .venv/bin/activate | |
| # Install requirements | |
| pip install -r $CONCRETE_PYTHON/requirements.txt | |
| pip install -r $CONCRETE_PYTHON/requirements.dev.txt | |
| # Build python bindings of concrete compiler | |
| cd $CONCRETE_COMPILER | |
| echo "Debug: ccache statistics (prior to the build):" && ccache -s | |
| make Python3_EXECUTABLE=$(which python) python-bindings | |
| echo "Debug: ccache statistics (after the build):" && ccache -s | |
| # Build wheel | |
| cd $CONCRETE_PYTHON | |
| rm -rf dist | |
| mkdir -p dist | |
| pip wheel -v --no-deps -w dist . | |
| delocate-wheel -v dist/*macos*.whl | |
| deactivate | |
| - name: Upload wheel | |
| uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 | |
| with: | |
| name: ${{ format('wheel-{0}-{1}', matrix.python-version, matrix.runs-on) }} | |
| path: frontends/concrete-python/dist/*macos*.whl | |
| retention-days: 3 | |
| hash: | |
| # Generate hashes for the wheels, used later for provenance. | |
| needs: [build-linux-x86, build-macos] | |
| runs-on: ubuntu-latest | |
| outputs: | |
| hash: ${{ steps.hash.outputs.hash }} | |
| steps: | |
| - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
| with: | |
| path: frontends/concrete-python/dist | |
| pattern: wheel-* | |
| merge-multiple: true | |
| - name: generate hash | |
| id: hash | |
| run: cd frontends/concrete-python/dist && echo "hash=$(sha256sum *.whl | base64 -w0)" >> $GITHUB_OUTPUT | |
| provenance: | |
| needs: [hash] | |
| permissions: | |
| actions: read | |
| id-token: write | |
| contents: write | |
| # Can't pin with hash due to how this workflow works. | |
| uses: slsa-framework/slsa-github-generator/.github/workflows/[email protected] | |
| with: | |
| base64-subjects: ${{ needs.hash.outputs.hash }} | |
| push: | |
| needs: [release-checks, build-linux-x86, build-macos, provenance] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
| with: | |
| path: wheels | |
| pattern: 'wheel-*' | |
| merge-multiple: true | |
| - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
| with: | |
| pattern: '*.intoto.jsonl' | |
| # When building a new public tag, create a new draft release. | |
| - name: create draft release | |
| if: ${{ env.RELEASE_TYPE == 'public'}} | |
| run: | | |
| export TAG=$(git describe --tags --abbrev=0) | |
| echo $TAG | |
| gh release create --draft --repo ${{ github.repository }} \ | |
| --verify-tag $TAG \ | |
| --title $TAG \ | |
| wheels/* *.intoto.jsonl/* | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| - name: Upload wheels to S3 | |
| if: ${{ env.RELEASE_TYPE == 'public' || env.RELEASE_TYPE == 'nightly' }} | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_IAM_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_IAM_KEY }} | |
| AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} | |
| S3_BUCKET_NAME: ${{ secrets.AWS_S3_PYPI_BUCKET_NAME }} | |
| CLOUDFRONT_DISTRIBUTION_ID: ${{ secrets.AWS_CLOUDFRONT_PYPI_DISTRIBUTION_ID }} | |
| run: | | |
| pip install boto3 bigtree | |
| # upload wheels | |
| aws s3 sync ./wheels/ s3://${S3_BUCKET_NAME}/cpu/concrete-python | |
| # update indexes and invalidate cloudfront cache | |
| python .github/workflows/scripts/s3_update_html_indexes.py | |
| - name: Start pushing Docker images | |
| if: ${{ env.RELEASE_TYPE == 'public' }} | |
| run: | | |
| export TAG=$(git describe --tags --abbrev=0) | |
| curl -L \ | |
| -X POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| https://api.github.com/repos/zama-ai/concrete/actions/workflows/concrete_python_push_docker_image.yml/dispatches \ | |
| -d "{\"ref\": \"$TAG\", \"inputs\": {\"tag\":\"v$TAG\"}}" | |
| test-linux-x86: | |
| needs: [build-linux-x86] | |
| continue-on-error: true | |
| strategy: | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] | |
| runs-on: ${{ github.event.inputs.runner_name }} | |
| steps: | |
| - uses: actions-rust-lang/setup-rust-toolchain@11df97af8e8102fd60b60a77dfbf58d40cd843b8 # v1.10.1 | |
| - name: Setup Python | |
| uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Download wheels | |
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
| with: | |
| name: ${{ format('wheel-{0}-linux-x86', matrix.python-version) }} | |
| path: ${{ format('wheel-{0}-linux-x86', matrix.python-version) }} | |
| - name: Checkout the repository | |
| uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 | |
| with: | |
| submodules: recursive | |
| path: repo | |
| - name: Test wheel | |
| run: | | |
| WHEEL_DIR=$(pwd)/${{ format('wheel-{0}-linux-x86', matrix.python-version) }} | |
| CONCRETE_PYTHON=$(pwd)/repo/frontends/concrete-python | |
| # Initialize an empty test environment | |
| cd $(mktemp -d) | |
| python -m venv .testenv && source .testenv/bin/activate | |
| # Install the concrete-python wheel | |
| pip install $WHEEL_DIR/*.whl | |
| # Install extra requirements for tests | |
| sudo apt update -y | |
| sudo apt install -y graphviz libgraphviz-dev | |
| pip install -r $CONCRETE_PYTHON/requirements.extra-full.txt | |
| pip install -r $CONCRETE_PYTHON/requirements.dev.txt | |
| # TODO - check for version | |
| # Copy test files | |
| cp -R $CONCRETE_PYTHON/tests . | |
| cp -R $CONCRETE_PYTHON/pytest.ini . | |
| cp $CONCRETE_PYTHON/Makefile . | |
| # Running tests | |
| make tfhers-utils | |
| pytest tests -svv -n auto | |
| test-macos: | |
| needs: [build-macos] | |
| continue-on-error: true | |
| strategy: | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] | |
| runs-on: ["aws-mac1-metal", "aws-mac2-metal"] | |
| runs-on: ${{ matrix.runs-on }} | |
| steps: | |
| - name: Download wheels | |
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
| with: | |
| name: ${{ format('wheel-{0}-{1}', matrix.python-version, matrix.runs-on) }} | |
| path: ${{ format('wheel-{0}-{1}', matrix.python-version, matrix.runs-on) }} | |
| - name: Checkout the repository | |
| uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 | |
| with: | |
| submodules: recursive | |
| path: repo | |
| - name: Test wheel | |
| run: | | |
| WHEEL_DIR=$(pwd)/${{ format('wheel-{0}-{1}', matrix.python-version, matrix.runs-on) }} | |
| CONCRETE_PYTHON=$(pwd)/repo/frontends/concrete-python | |
| PYTHON=${{ format('python{0}', matrix.python-version) }} | |
| # Initialize an empty test environment | |
| export TEST_TMP_DIR=$(mktemp -d) | |
| echo "TEST_TMP_DIR=$TEST_TMP_DIR" >> "${GITHUB_ENV}" | |
| cd $TEST_TMP_DIR | |
| # Activate virtual environment | |
| $PYTHON -m venv .testenv && source .testenv/bin/activate | |
| # Install extra requirements for tests | |
| pip install $WHEEL_DIR/*macos*.whl | |
| pip install -r $CONCRETE_PYTHON/requirements.dev.txt | |
| # MacOS x86 have conflict between our OpenMP library, and one from torch | |
| # we fix it by using a single one (from torch) | |
| # see discussion: https://discuss.python.org/t/conflicting-binary-extensions-in-different-packages/25332/8 | |
| export SITE_PACKAGES=$(pwd)/.testenv/lib/$PYTHON_VERSION/site-packages/ | |
| find $SITE_PACKAGES -not \( -path $SITE_PACKAGES/concrete -prune \) -name 'lib*omp5.dylib' -or -name 'lib*omp.dylib' | xargs -n 1 ln -f -s $SITE_PACKAGES/concrete/.dylibs/libomp.dylib | |
| # Copy test files | |
| cp -R $CONCRETE_PYTHON/tests . | |
| cp -R $CONCRETE_PYTHON/pytest.ini . | |
| cp $CONCRETE_PYTHON/Makefile . | |
| # Running tests | |
| make tfhers-utils | |
| mkdir ./KeySetCache | |
| pytest tests -svv -n auto --key-cache "./KeySetCache" -m "not dataflow and not graphviz" | |
| - name: Cleanup host | |
| if: success() || failure() | |
| run: | | |
| rm -rf $TEST_TMP_DIR |