concrete-python release #238
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Concrete Python Release | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| instance_id: | |
| description: 'Instance ID' | |
| type: string | |
| instance_image_id: | |
| description: 'Instance AMI ID' | |
| type: string | |
| instance_type: | |
| description: 'Instance product type' | |
| type: string | |
| runner_name: | |
| description: 'Action runner name' | |
| type: string | |
| request_id: | |
| description: 'Slab request ID' | |
| type: string | |
| user_inputs: | |
| description: 'either "nightly" or "public" to specify the release type' | |
| required: true | |
| default: 'nightly' | |
| type: string | |
| env: | |
| DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler | |
| GLIB_VER: 2_28 | |
| RELEASE_TYPE: ${{ inputs.user_inputs }} | |
| jobs: | |
| linux-x86: | |
| continue-on-error: true | |
| strategy: | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10", "3.11"] | |
| runs-on: ${{ github.event.inputs.runner_name }} | |
| steps: | |
| - name: Log instance configuration | |
| run: | | |
| echo "IDs: ${{ inputs.instance_id }}" | |
| echo "AMI: ${{ inputs.instance_image_id }}" | |
| echo "Type: ${{ inputs.instance_type }}" | |
| echo "Request ID: ${{ inputs.request_id }}" | |
| echo "User Inputs: ${{ inputs.user_inputs }}" | |
| - name: Set up SSH agent | |
| uses: webfactory/[email protected] | |
| with: | |
| ssh-private-key: ${{ secrets.CONCRETE_CI_SSH_PRIVATE }} | |
| - name: Set up GitHub environment | |
| run: | | |
| echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" | |
| #echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK)" >> "${GITHUB_ENV}" | |
| echo "SSH_AUTH_SOCK_DIR=$(dirname $SSH_AUTH_SOCK)" >> "${GITHUB_ENV}" | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} | |
| - name: Set release version (nightly) | |
| if: ${{ env.RELEASE_TYPE == 'nightly' }} | |
| run: | | |
| VERSION=$(date +"%Y.%m.%d") | |
| echo "__version__ = \"$VERSION\"" >| frontends/concrete-python/version.txt | |
| git tag nightly-$VERSION || true | |
| git push origin nightly-$VERSION || true | |
| - name: Set release version (public) | |
| if: ${{ env.RELEASE_TYPE == 'public' }} | |
| run: echo "__version__ = \"`git describe --tags --abbrev=0 | grep -e '[0-9].*' -o`\"" >| frontends/concrete-python/version.txt | |
| - name: Expose release version from Python | |
| run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py | |
| - name: Create build directory | |
| run: mkdir build | |
| - name: Build wheel | |
| uses: addnab/docker-run-action@v3 | |
| id: build-compiler-bindings | |
| with: | |
| registry: ghcr.io | |
| image: ${{ env.DOCKER_IMAGE_TEST }} | |
| username: ${{ secrets.GHCR_LOGIN }} | |
| password: ${{ secrets.GHCR_PASSWORD }} | |
| options: >- | |
| -v ${{ github.workspace }}:/concrete | |
| -v ${{ github.workspace }}/build:/build | |
| -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket | |
| -e SSH_AUTH_SOCK=/ssh.socket | |
| ${{ env.DOCKER_GPU_OPTION }} | |
| shell: bash | |
| run: | | |
| set -e | |
| rm -rf /build/* | |
| export PYTHON=${{ format('python{0}', matrix.python-version) }} | |
| echo "Using $PYTHON" | |
| dnf -y install graphviz graphviz-devel | |
| cd /concrete/frontends/concrete-python | |
| make PYTHON=$PYTHON venv | |
| source .venv/bin/activate | |
| cd /concrete/compilers/concrete-compiler/compiler | |
| make BUILD_DIR=/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=ON Python3_EXECUTABLE=$(which python) python-bindings | |
| echo "Debug: ccache statistics (after the build):" | |
| ccache -s | |
| cd /concrete/frontends/concrete-python | |
| export COMPILER_BUILD_DIRECTORY="/build" | |
| make whl | |
| deactivate | |
| - name: Upload wheel | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ format('wheel-{0}-linux-x86', matrix.python-version) }} | |
| path: frontends/concrete-python/dist/*manylinux*.whl | |
| retention-days: 3 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Test wheel | |
| run: | | |
| set -e | |
| export CONCRETE_PYTHON=$(pwd)/frontends/concrete-python | |
| cd $(mktemp -d) | |
| python -m venv .testenv | |
| source .testenv/bin/activate | |
| sudo apt install graphviz graphviz-devel | |
| pip install $CONCRETE_PYTHON/dist/*manylinux*.whl | |
| pip install -r $CONCRETE_PYTHON/requirements.dev.txt | |
| expected_version=$(cat "$CONCRETE_PYTHON/version.txt") | |
| actual_version=$(python -c "from concrete import fhe; print(f'__version__ = \"{fhe.__version__}\"')") | |
| if [ "$actual_version" != "$expected_version" ]; then | |
| echo "Release version is not properly set" | |
| exit 1 | |
| fi | |
| cp -R $CONCRETE_PYTHON/tests . | |
| cp -R $CONCRETE_PYTHON/pytest.ini . | |
| pytest -svv -n auto tests --key-cache /tmp/KeySetCache | |
| rm -rf /tmp/KeySetCache | |
| deactivate | |
| macos: | |
| continue-on-error: true | |
| strategy: | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10", "3.11"] | |
| runs-on: ["aws-mac1-metal", "aws-mac2-metal"] | |
| runs-on: ${{ matrix.runs-on }} | |
| steps: | |
| - name: Set up SSH agent | |
| uses: webfactory/[email protected] | |
| with: | |
| ssh-private-key: ${{ secrets.CONCRETE_CI_SSH_PRIVATE }} | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} | |
| - name: Install OS Dependencies | |
| run: | | |
| brew install ninja ccache graphviz | |
| - name: Install Rust | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| toolchain: nightly | |
| override: true | |
| - name: Set release version (nightly) | |
| if: ${{ env.RELEASE_TYPE == 'nightly' }} | |
| run: echo "__version__ = \"$(date +"%Y.%m.%d")\"" >| frontends/concrete-python/version.txt | |
| - name: Set release version (public) | |
| if: ${{ env.RELEASE_TYPE == 'public' }} | |
| run: echo "__version__ = \"`git describe --tags --abbrev=0 | grep -e '[0-9].*' -o`\"" >| frontends/concrete-python/version.txt | |
| - name: Expose release version from Python | |
| run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py | |
| - name: Create build directory | |
| run: mkdir build | |
| - name: Build wheel | |
| run: | | |
| set -e | |
| cd frontends/concrete-python | |
| # Setup pkg-config to find OpenBLAS (scipy need it) | |
| export PKG_CONFIG_PATH="/opt/homebrew/opt/openblas/lib/pkgconfig" | |
| rm -rf .venv | |
| ${{ format('python{0}', matrix.python-version) }} -m venv .venv | |
| . .venv/bin/activate | |
| CFLAGS=-I$(brew --prefix graphviz)/include LDFLAGS=-L$(brew --prefix graphviz)/lib pip --no-cache-dir install pygraphviz | |
| pip install -r requirements.dev.txt | |
| pip install -r requirements.txt | |
| cd $GITHUB_WORKSPACE/compilers/concrete-compiler/compiler | |
| echo "Debug: ccache statistics (prior to the build):" | |
| ccache -s | |
| make Python3_EXECUTABLE=$(which python) python-bindings | |
| echo "Debug: ccache statistics (after the build):" | |
| ccache -s | |
| export COMPILER_BUILD_DIRECTORY=$(pwd)/build | |
| cd $GITHUB_WORKSPACE/frontends/concrete-python | |
| rm -rf dist | |
| mkdir -p dist | |
| pip wheel -v --no-deps -w dist . | |
| delocate-wheel -v dist/*macos*.whl | |
| deactivate | |
| - name: Upload wheel | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ format('wheel-{0}-{1}', matrix.python-version, matrix.runs-on) }} | |
| path: frontends/concrete-python/dist/*macos*.whl | |
| retention-days: 3 | |
| - name: Test wheel | |
| run: | | |
| set -e | |
| export CONCRETE_PYTHON=$(pwd)/frontends/concrete-python | |
| export TEST_TMP_DIR=$(mktemp -d) | |
| cd $TEST_TMP_DIR | |
| export PYTHON_VERSION=${{ format('python{0}', matrix.python-version) }} | |
| $PYTHON_VERSION -m venv .testenv | |
| source .testenv/bin/activate | |
| CFLAGS=-I$(brew --prefix graphviz)/include LDFLAGS=-L$(brew --prefix graphviz)/lib pip --no-cache-dir install pygraphviz | |
| pip install $CONCRETE_PYTHON/dist/*macos*.whl | |
| pip install -r $CONCRETE_PYTHON/requirements.dev.txt | |
| expected_version=$(cat "$CONCRETE_PYTHON/version.txt") | |
| actual_version=$(python -c "from concrete import fhe; print(f'__version__ = \"{fhe.__version__}\"')") | |
| if [ "$actual_version" != "$expected_version" ]; then | |
| echo "Release version is not properly set" | |
| exit 1 | |
| fi | |
| # MacOS x86 have conflict between our OpenMP library, and one from torch | |
| # we fix it by using a single one (from torch) | |
| # see discussion: https://discuss.python.org/t/conflicting-binary-extensions-in-different-packages/25332/8 | |
| export SITE_PACKAGES=$(pwd)/.testenv/lib/$PYTHON_VERSION/site-packages/ | |
| export CONCRETE_LIB_OMP=$SITE_PACKAGES/concrete/.dylibs/libomp.dylib | |
| # Where is libiomp? Possibilities ordered in increasing priorities | |
| export TORCH_LIB_IOMP="this shouldn't exist" | |
| ls $SITE_PACKAGES/torch/lib/libiomp5.dylib && export TORCH_LIB_IOMP=$SITE_PACKAGES/torch/lib/libiomp5.dylib | |
| ls $SITE_PACKAGES/torch/.dylibs/libiomp5.dylib && export TORCH_LIB_IOMP=$SITE_PACKAGES/torch/.dylibs/libiomp5.dylib | |
| ls $SITE_PACKAGES/functorch/.dylibs/libiomp5.dylib && export TORCH_LIB_IOMP=$SITE_PACKAGES/functorch/.dylibs/libiomp5.dylib | |
| ls $TORCH_LIB_IOMP \ | |
| && rm $CONCRETE_LIB_OMP \ | |
| && ln -s $TORCH_LIB_IOMP $CONCRETE_LIB_OMP | |
| cp -R $CONCRETE_PYTHON/tests . | |
| cp -R $CONCRETE_PYTHON/pytest.ini . | |
| pytest tests -svv -n auto -m "not dataflow" --key-cache /tmp/KeySetCache | |
| rm -rf /tmp/KeySetCache | |
| rm -rf $TEST_TMP_DIR | |
| push: | |
| needs: [linux-x86, macos] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| continue-on-error: ${{ env.RELEASE_TYPE == 'nightly' }} | |
| with: | |
| name: wheel-3.8-linux-x86 | |
| path: wheels | |
| - uses: actions/download-artifact@v4 | |
| continue-on-error: ${{ env.RELEASE_TYPE == 'nightly' }} | |
| with: | |
| name: wheel-3.9-linux-x86 | |
| path: wheels | |
| - uses: actions/download-artifact@v4 | |
| continue-on-error: ${{ env.RELEASE_TYPE == 'nightly' }} | |
| with: | |
| name: wheel-3.10-linux-x86 | |
| path: wheels | |
| - uses: actions/download-artifact@v4 | |
| continue-on-error: ${{ env.RELEASE_TYPE == 'nightly' }} | |
| with: | |
| name: wheel-3.11-linux-x86 | |
| path: wheels | |
| - uses: actions/download-artifact@v4 | |
| continue-on-error: ${{ env.RELEASE_TYPE == 'nightly' }} | |
| with: | |
| name: wheel-3.8-aws-mac1-metal | |
| path: wheels | |
| - uses: actions/download-artifact@v4 | |
| continue-on-error: ${{ env.RELEASE_TYPE == 'nightly' }} | |
| with: | |
| name: wheel-3.9-aws-mac1-metal | |
| path: wheels | |
| - uses: actions/download-artifact@v4 | |
| continue-on-error: ${{ env.RELEASE_TYPE == 'nightly' }} | |
| with: | |
| name: wheel-3.10-aws-mac1-metal | |
| path: wheels | |
| - uses: actions/download-artifact@v4 | |
| continue-on-error: ${{ env.RELEASE_TYPE == 'nightly' }} | |
| with: | |
| name: wheel-3.11-aws-mac1-metal | |
| path: wheels | |
| - uses: actions/download-artifact@v4 | |
| continue-on-error: ${{ env.RELEASE_TYPE == 'nightly' }} | |
| with: | |
| name: wheel-3.8-aws-mac2-metal | |
| path: wheels | |
| - uses: actions/download-artifact@v4 | |
| continue-on-error: ${{ env.RELEASE_TYPE == 'nightly' }} | |
| with: | |
| name: wheel-3.9-aws-mac2-metal | |
| path: wheels | |
| - uses: actions/download-artifact@v4 | |
| continue-on-error: ${{ env.RELEASE_TYPE == 'nightly' }} | |
| with: | |
| name: wheel-3.10-aws-mac2-metal | |
| path: wheels | |
| - uses: actions/download-artifact@v4 | |
| continue-on-error: ${{ env.RELEASE_TYPE == 'nightly' }} | |
| with: | |
| name: wheel-3.11-aws-mac2-metal | |
| path: wheels | |
| - name: Push wheels to internal PyPI (nightly) | |
| if: ${{ env.RELEASE_TYPE == 'nightly' }} | |
| run: | | |
| pip install twine==4.0.2 | |
| twine upload wheels/*.whl \ | |
| -u "${{ secrets.INTERNAL_PYPI_USER }}" \ | |
| -p "${{ secrets.INTERNAL_PYPI_PASSWORD }}" \ | |
| --repository-url "${{ secrets.INTERNAL_PYPI_URL }}" | |
| - name: Push wheels to public PyPI (public) | |
| if: ${{ env.RELEASE_TYPE == 'public' }} | |
| run: | | |
| pip install twine==4.0.2 | |
| twine upload wheels/*.whl \ | |
| -u "${{ secrets.PUBLIC_PYPI_USER }}" \ | |
| -p "${{ secrets.PUBLIC_PYPI_PASSWORD }}" \ | |
| -r pypi | |
| - name: Start pushing Docker images | |
| if: ${{ env.RELEASE_TYPE == 'public' }} | |
| run: | | |
| export TAG=$(git describe --tags --abbrev=0) | |
| curl -L \ | |
| -X POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "Authorization: Bearer ${{ secrets.CONCRETE_ACTIONS_TOKEN }}" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| https://api.github.com/repos/zama-ai/concrete/actions/workflows/concrete_python_push_docker_image.yml/dispatches \ | |
| -d '{"ref":"$TAG","inputs":{"tag":"$TAG"}}' |