Integer benchmarks #26
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Run all integer benchmarks on an AWS instance and return parsed results to Slab CI bot. | |
| name: Integer benchmarks | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| all_precisions: | |
| description: "Run all precisions" | |
| type: boolean | |
| default: false | |
| bench_type: | |
| description: "Benchmarks type" | |
| type: choice | |
| default: latency | |
| options: | |
| - latency | |
| - throughput | |
| - both | |
| schedule: | |
| # Weekly benchmarks will be triggered each Saturday at 1a.m. | |
| - cron: '0 1 * * 6' | |
| # Quarterly benchmarks will be triggered right before end of quarter, the 25th of the current month at 4a.m. | |
| # These benchmarks are far longer to execute hence the reason to run them only four time a year. | |
| - cron: '0 4 25 MAR,JUN,SEP,DEC *' | |
| env: | |
| CARGO_TERM_COLOR: always | |
| RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json | |
| ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| RUST_BACKTRACE: "full" | |
| RUST_MIN_STACK: "8388608" | |
| SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} | |
| SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png | |
| SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} | |
| FAST_BENCH: TRUE | |
| jobs: | |
| prepare-matrix: | |
| name: Prepare operations matrix | |
| runs-on: ubuntu-latest | |
| if: github.event_name != 'schedule' || | |
| (github.event_name == 'schedule' && github.repository == 'zama-ai/tfhe-rs') | |
| outputs: | |
| op_flavor: ${{ steps.set_op_flavor.outputs.op_flavor }} | |
| bench_type: ${{ steps.set_bench_type.outputs.bench_type }} | |
| steps: | |
| - name: Weekly benchmarks | |
| if: github.event.schedule == '0 1 * * 6' | |
| run: | | |
| echo "OP_FLAVOR=[\"default\"]" >> "${GITHUB_ENV}" | |
| - name: Quarterly benchmarks | |
| if: github.event.schedule == '0 4 25 MAR,JUN,SEP,DEC *' | |
| run: | | |
| echo "OP_FLAVOR=[\"default\", \"smart\", \"unchecked\", \"misc\"]" >> "${GITHUB_ENV}" | |
| - name: Set benchmark types | |
| if: github.event_name == 'workflow_dispatch' | |
| run: | | |
| echo "OP_FLAVOR=[\"default\"]" >> "${GITHUB_ENV}" | |
| if [[ "${{ inputs.bench_type }}" == "both" ]]; then | |
| echo "BENCH_TYPE=[\"latency\", \"throughput\"]" >> "${GITHUB_ENV}" | |
| else | |
| echo "BENCH_TYPE=[\"${{ inputs.bench_type }}\"]" >> "${GITHUB_ENV}" | |
| fi | |
| - name: Default benchmark type | |
| if: github.event_name != 'workflow_dispatch' | |
| run: | | |
| echo "BENCH_TYPE=[\"latency\"]" >> "${GITHUB_ENV}" | |
| - name: Set operation flavor output | |
| id: set_op_flavor | |
| run: | | |
| echo "op_flavor=${{ toJSON(env.OP_FLAVOR) }}" >> "${GITHUB_OUTPUT}" | |
| - name: Set benchmark types output | |
| id: set_bench_type | |
| run: | | |
| echo "bench_type=${{ toJSON(env.BENCH_TYPE) }}" >> "${GITHUB_OUTPUT}" | |
| setup-instance: | |
| name: Setup instance (integer-benchmarks) | |
| needs: prepare-matrix | |
| runs-on: ubuntu-latest | |
| outputs: | |
| runner-name: ${{ steps.start-instance.outputs.label }} | |
| steps: | |
| - name: Start instance | |
| id: start-instance | |
| uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac | |
| with: | |
| mode: start | |
| github-token: ${{ secrets.SLAB_ACTION_TOKEN }} | |
| slab-url: ${{ secrets.SLAB_BASE_URL }} | |
| job-secret: ${{ secrets.JOB_SECRET }} | |
| backend: aws | |
| profile: bench | |
| integer-benchmarks: | |
| name: Execute integer benchmarks for all operations flavor | |
| needs: [ prepare-matrix, setup-instance ] | |
| runs-on: ${{ needs.setup-instance.outputs.runner-name }} | |
| concurrency: | |
| group: ${{ github.workflow }}_${{ github.ref }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} | |
| continue-on-error: true | |
| timeout-minutes: 1440 # 24 hours | |
| strategy: | |
| max-parallel: 1 | |
| matrix: | |
| command: [ integer, integer_multi_bit] | |
| op_flavor: ${{ fromJson(needs.prepare-matrix.outputs.op_flavor) }} | |
| bench_type: ${{ fromJSON(needs.prepare-matrix.outputs.bench_type) }} | |
| steps: | |
| - name: Checkout tfhe-rs repo with tags | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 | |
| with: | |
| fetch-depth: 0 | |
| token: ${{ secrets.FHE_ACTIONS_TOKEN }} | |
| - name: Get benchmark details | |
| run: | | |
| { | |
| echo "BENCH_DATE=$(date --iso-8601=seconds)"; | |
| echo "COMMIT_DATE=$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})"; | |
| echo "COMMIT_HASH=$(git describe --tags --dirty)"; | |
| } >> "${GITHUB_ENV}" | |
| - name: Install rust | |
| uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203 | |
| with: | |
| toolchain: nightly | |
| - name: Checkout Slab repo | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 | |
| with: | |
| repository: zama-ai/slab | |
| path: slab | |
| token: ${{ secrets.FHE_ACTIONS_TOKEN }} | |
| - name: Should run benchmarks with all precisions | |
| if: inputs.all_precisions | |
| run: | | |
| echo "FAST_BENCH=FALSE" >> "${GITHUB_ENV}" | |
| - name: Run benchmarks with AVX512 | |
| run: | | |
| make BENCH_OP_FLAVOR=${{ matrix.op_flavor }} BENCH_TYPE=${{ matrix.bench_type }} bench_${{ matrix.command }} | |
| # Run these benchmarks only once per benchmark type | |
| - name: Run compression benchmarks with AVX512 | |
| if: matrix.op_flavor == 'default' && matrix.command == 'integer' | |
| run: | | |
| make BENCH_TYPE=${{ matrix.bench_type }} bench_integer_compression | |
| - name: Parse results | |
| run: | | |
| python3 ./ci/benchmark_parser.py target/criterion ${{ env.RESULTS_FILENAME }} \ | |
| --database tfhe_rs \ | |
| --hardware "hpc7a.96xlarge" \ | |
| --project-version "${{ env.COMMIT_HASH }}" \ | |
| --branch ${{ github.ref_name }} \ | |
| --commit-date "${{ env.COMMIT_DATE }}" \ | |
| --bench-date "${{ env.BENCH_DATE }}" \ | |
| --walk-subdirs \ | |
| --name-suffix avx512 \ | |
| --bench-type ${{ matrix.bench_type }} | |
| - name: Upload parsed results artifact | |
| uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 | |
| with: | |
| name: ${{ github.sha }}_${{ matrix.command }}_${{ matrix.op_flavor }}_${{ matrix.bench_type }} | |
| path: ${{ env.RESULTS_FILENAME }} | |
| - name: Send data to Slab | |
| shell: bash | |
| run: | | |
| python3 slab/scripts/data_sender.py ${{ env.RESULTS_FILENAME }} "${{ secrets.JOB_SECRET }}" \ | |
| --slab-url "${{ secrets.SLAB_URL }}" | |
| - name: Slack Notification | |
| if: ${{ failure() }} | |
| continue-on-error: true | |
| uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 | |
| env: | |
| SLACK_COLOR: ${{ job.status }} | |
| SLACK_MESSAGE: "Integer full benchmarks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" | |
| teardown-instance: | |
| name: Teardown instance (integer-benchmarks) | |
| if: ${{ always() && needs.setup-instance.result != 'skipped' }} | |
| needs: [ setup-instance, integer-benchmarks ] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Stop instance | |
| id: stop-instance | |
| uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac | |
| with: | |
| mode: stop | |
| github-token: ${{ secrets.SLAB_ACTION_TOKEN }} | |
| slab-url: ${{ secrets.SLAB_BASE_URL }} | |
| job-secret: ${{ secrets.JOB_SECRET }} | |
| label: ${{ needs.setup-instance.outputs.runner-name }} | |
| - name: Slack Notification | |
| if: ${{ failure() }} | |
| continue-on-error: true | |
| uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 | |
| env: | |
| SLACK_COLOR: ${{ job.status }} | |
| SLACK_MESSAGE: "Instance teardown (integer-benchmarks) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" |