Shortint full benchmarks #40
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Run all shortint benchmarks on an AWS instance and return parsed results to Slab CI bot. | |
| name: Shortint full benchmarks | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| instance_id: | |
| description: "Instance ID" | |
| type: string | |
| instance_image_id: | |
| description: "Instance AMI ID" | |
| type: string | |
| instance_type: | |
| description: "Instance product type" | |
| type: string | |
| runner_name: | |
| description: "Action runner name" | |
| type: string | |
| request_id: | |
| description: "Slab request ID" | |
| type: string | |
| # This input is not used in this workflow but still mandatory since a calling workflow could | |
| # use it. If a triggering command include a user_inputs field, then the triggered workflow | |
| # must include this very input, otherwise the workflow won't be called. | |
| # See start_full_benchmarks.yml as example. | |
| user_inputs: | |
| description: "Type of benchmarks to run" | |
| type: string | |
| default: "weekly_benchmarks" | |
| env: | |
| CARGO_TERM_COLOR: always | |
| RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json | |
| ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| RUST_BACKTRACE: "full" | |
| RUST_MIN_STACK: "8388608" | |
| jobs: | |
| shortint-benchmarks: | |
| name: Execute shortint benchmarks for all operations flavor | |
| runs-on: ${{ github.event.inputs.runner_name }} | |
| if: ${{ !cancelled() }} | |
| strategy: | |
| max-parallel: 1 | |
| matrix: | |
| op_flavor: [ default, smart, unchecked ] | |
| steps: | |
| - name: Instance configuration used | |
| run: | | |
| echo "IDs: ${{ inputs.instance_id }}" | |
| echo "AMI: ${{ inputs.instance_image_id }}" | |
| echo "Type: ${{ inputs.instance_type }}" | |
| echo "Request ID: ${{ inputs.request_id }}" | |
| - name: Checkout tfhe-rs repo with tags | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 | |
| with: | |
| fetch-depth: 0 | |
| - name: Get benchmark details | |
| run: | | |
| { | |
| echo "BENCH_DATE=$(date --iso-8601=seconds)"; | |
| echo "COMMIT_DATE=$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})"; | |
| echo "COMMIT_HASH=$(git describe --tags --dirty)"; | |
| } >> "${GITHUB_ENV}" | |
| - name: Set up home | |
| # "Install rust" step require root user to have a HOME directory which is not set. | |
| run: | | |
| echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" | |
| - name: Install rust | |
| uses: dtolnay/rust-toolchain@be73d7920c329f220ce78e0234b8f96b7ae60248 | |
| with: | |
| toolchain: nightly | |
| - name: Checkout Slab repo | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 | |
| with: | |
| repository: zama-ai/slab | |
| path: slab | |
| token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} | |
| - name: Run benchmarks with AVX512 | |
| run: | | |
| make BENCH_OP_FLAVOR=${{ matrix.op_flavor }} bench_shortint | |
| - name: Parse results | |
| run: | | |
| COMMIT_DATE="$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})" | |
| COMMIT_HASH="$(git describe --tags --dirty)" | |
| python3 ./ci/benchmark_parser.py target/criterion ${{ env.RESULTS_FILENAME }} \ | |
| --database tfhe_rs \ | |
| --hardware ${{ inputs.instance_type }} \ | |
| --project-version "${COMMIT_HASH}" \ | |
| --branch ${{ github.ref_name }} \ | |
| --commit-date "${COMMIT_DATE}" \ | |
| --bench-date "${{ env.BENCH_DATE }}" \ | |
| --walk-subdirs \ | |
| --name-suffix avx512 \ | |
| --throughput | |
| # This small benchmark needs to be executed only once. | |
| - name: Measure key sizes | |
| if: matrix.op_flavor == 'default' | |
| run: | | |
| make measure_shortint_key_sizes | |
| - name: Parse key sizes results | |
| if: matrix.op_flavor == 'default' | |
| run: | | |
| python3 ./ci/benchmark_parser.py tfhe/shortint_key_sizes.csv ${{ env.RESULTS_FILENAME }} \ | |
| --key-sizes \ | |
| --append-results | |
| - name: Upload parsed results artifact | |
| uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 | |
| with: | |
| name: ${{ github.sha }}_shortint_${{ matrix.op_flavor }} | |
| path: ${{ env.RESULTS_FILENAME }} | |
| - name: Send data to Slab | |
| shell: bash | |
| run: | | |
| echo "Computing HMac on results file" | |
| SIGNATURE="$(slab/scripts/hmac_calculator.sh ${{ env.RESULTS_FILENAME }} '${{ secrets.JOB_SECRET }}')" | |
| echo "Sending results to Slab..." | |
| curl -v -k \ | |
| -H "Content-Type: application/json" \ | |
| -H "X-Slab-Repository: ${{ github.repository }}" \ | |
| -H "X-Slab-Command: store_data_v2" \ | |
| -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \ | |
| -d @${{ env.RESULTS_FILENAME }} \ | |
| ${{ secrets.SLAB_URL }} | |
| slack-notification: | |
| name: Slack Notification | |
| runs-on: ${{ github.event.inputs.runner_name }} | |
| if: ${{ failure() }} | |
| needs: shortint-benchmarks | |
| steps: | |
| - name: Notify | |
| continue-on-error: true | |
| uses: rtCamp/action-slack-notify@b24d75fe0e728a4bf9fc42ee217caa686d141ee8 | |
| env: | |
| SLACK_COLOR: ${{ job.status }} | |
| SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} | |
| SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png | |
| SLACK_MESSAGE: "Shortint full benchmarks failed. (${{ env.ACTION_RUN_URL }})" | |
| SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} |