Compiler - Build and Test (CPU) #2381
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Compiler - Build and Test (CPU) | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| instance_id: | |
| description: 'Instance ID' | |
| type: string | |
| instance_image_id: | |
| description: 'Instance AMI ID' | |
| type: string | |
| instance_type: | |
| description: 'Instance product type' | |
| type: string | |
| runner_name: | |
| description: 'Action runner name' | |
| type: string | |
| request_id: | |
| description: 'Slab request ID' | |
| type: string | |
| matrix_item: | |
| description: 'Build matrix item' | |
| type: string | |
| # concurrency: | |
| # group: compiler_build_and_test_cpu-${{ github.ref }} | |
| # cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} | |
| env: | |
| DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler | |
| GLIB_VER: 2_28 | |
| jobs: | |
| BuildAndTest: | |
| name: Build and test compiler in EC2 | |
| runs-on: ${{ github.event.inputs.runner_name }} | |
| if: ${{ !cancelled() }} | |
| steps: | |
| - name: Instance configuration used | |
| run: | | |
| echo "IDs: ${{ inputs.instance_id }}" | |
| echo "AMI: ${{ inputs.instance_image_id }}" | |
| echo "Type: ${{ inputs.instance_type }}" | |
| echo "Request ID: ${{ inputs.request_id }}" | |
| echo "Matrix item: ${{ inputs.matrix_item }}" | |
| - name: Set up env | |
| run: | | |
| echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" | |
| #echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK)" >> "${GITHUB_ENV}" | |
| echo "SSH_AUTH_SOCK_DIR=$(dirname $SSH_AUTH_SOCK)" >> "${GITHUB_ENV}" | |
| - name: Fetch repository | |
| uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 | |
| with: | |
| submodules: recursive | |
| token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} | |
| - name: Setup rust toolchain for concrete-cpu | |
| uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu | |
| - name: Create build dir | |
| run: mkdir build | |
| - name: Build compiler | |
| uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 | |
| id: build-compiler | |
| with: | |
| registry: ghcr.io | |
| image: ${{ env.DOCKER_IMAGE_TEST }} | |
| username: ${{ secrets.GHCR_LOGIN }} | |
| password: ${{ secrets.GHCR_PASSWORD }} | |
| options: >- | |
| -v ${{ github.workspace }}:/concrete | |
| -v ${{ github.workspace }}/build:/build | |
| -v ${{ github.workspace }}/wheels:/wheels | |
| -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket | |
| -e SSH_AUTH_SOCK=/ssh.socket | |
| ${{ env.DOCKER_GPU_OPTION }} | |
| shell: bash | |
| run: | | |
| rustup toolchain install nightly-2024-09-30 | |
| pip install mypy | |
| set -e | |
| cd /concrete/compilers/concrete-compiler/compiler | |
| rm -rf /build/* | |
| make DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build all | |
| echo "Debug: ccache statistics (after the build):" | |
| ccache -s | |
| - name: Build compiler Dialects docs and check diff | |
| uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 | |
| id: build-compiler-docs | |
| with: | |
| registry: ghcr.io | |
| image: ${{ env.DOCKER_IMAGE_TEST }} | |
| username: ${{ secrets.GHCR_LOGIN }} | |
| password: ${{ secrets.GHCR_PASSWORD }} | |
| options: >- | |
| -v ${{ github.workspace }}:/concrete | |
| -v ${{ github.workspace }}/build:/build | |
| -v ${{ github.workspace }}/wheels:/wheels | |
| -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket | |
| -e SSH_AUTH_SOCK=/ssh.socket | |
| ${{ env.DOCKER_GPU_OPTION }} | |
| shell: bash | |
| run: | | |
| set -e | |
| cd /concrete/compilers/concrete-compiler/compiler | |
| make BUILD_DIR=/build doc | |
| cd /build/tools/concretelang/docs/concretelang/ | |
| sed -i -e 's/\[TOC\]//' *Dialect.md | |
| for i in `ls *Dialect.md`; do diff $i /concrete/docs/explanations/$i; done; | |
| - name: Enable complete tests on push to main | |
| if: github.ref == 'refs/heads/main' | |
| run: echo "MINIMAL_TESTS=OFF" >> $GITHUB_ENV | |
| - name: Enable minimal tests otherwise | |
| if: github.ref != 'refs/heads/main' | |
| run: echo "MINIMAL_TESTS=ON" >> $GITHUB_ENV | |
| - name: Test compiler | |
| uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 | |
| with: | |
| registry: ghcr.io | |
| image: ${{ env.DOCKER_IMAGE_TEST }} | |
| username: ${{ secrets.GHCR_LOGIN }} | |
| password: ${{ secrets.GHCR_PASSWORD }} | |
| options: >- | |
| -v ${{ github.workspace }}:/concrete | |
| -v ${{ github.workspace }}/build:/build | |
| ${{ env.DOCKER_GPU_OPTION }} | |
| shell: bash | |
| run: | | |
| set -e | |
| rustup toolchain install nightly-2024-09-30 | |
| cd /concrete/compilers/concrete-compiler/compiler | |
| pip install pytest | |
| pip install mypy | |
| dnf install -y libzstd libzstd-devel | |
| sed "s/pytest/python -m pytest/g" -i Makefile | |
| mkdir -p /tmp/concrete_compiler/gpu_tests/ | |
| make MINIMAL_TESTS=${{ env.MINIMAL_TESTS }} DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build run-tests | |
| chmod -R ugo+rwx /tmp/KeySetCache | |
| - name: Analyze logs | |
| run: | | |
| cd build/gtest-parallel-logs/passed | |
| ls -1 | xargs grep -H "WARNING RETRY" | sed -e "s/.log.*//g" | uniq -c | sed -re "s/ *([0-9]*) (.*)/::warning ::Test \2 retried \1 times/g" | cat | |
| # - name: Archive python package | |
| # uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 | |
| # with: | |
| # name: concrete-compiler.whl | |
| # path: build/wheels/concrete_compiler-*-manylinux_{{ env.GLIB_VER }}_x86_64.whl | |
| # retention-days: 14 |