Benchmark #50
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| repository: | |
| type: string | |
| description: name of repository triggers this workflow, e.g. FluxML/NNlib.jl | |
| pr_id: | |
| type: string | |
| description: id of the pull request that triggers this workflow | |
| target_url: | |
| type: string | |
| description: url of target | |
| baseline_url: | |
| type: string | |
| description: url of baseline | |
| enable_gpu: | |
| type: boolean | |
| description: run gpu benchmarks | |
| jobs: | |
| Cache: | |
| runs-on: ubuntu-latest | |
| env: | |
| TARGET_URL: ${{ github.event.inputs.target_url || format('{0}#{1}', github.event.pull_request.head.repo.html_url, github.event.pull_request.head.sha) }} | |
| BASELINE_URL: ${{ github.event.inputs.baseline_url || format('{0}#{1}', github.event.pull_request.base.repo.html_url, github.event.pull_request.base.sha) }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - | |
| name: Print target&baseline repo url | |
| run: | | |
| echo "Target repo url: $TARGET_URL" | |
| echo "Baseline repo url: $BASELINE_URL" | |
| - | |
| name: Get current time | |
| run: echo "TIME=$(date +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_ENV | |
| - | |
| id: restore-cache | |
| uses: actions/cache/restore@v3 | |
| with: | |
| path: ~/.julia | |
| key: bm-${{ github.run_id }}-${{ github.run_attempt }}-${{ env.TARGET_URL }}-${{ env.BASELINE_URL }}-${{ env.TIME }} | |
| restore-keys: bm-${{ github.run_id }}-${{ github.run_attempt }}-${{ env.TARGET_URL }}-${{ env.BASELINE_URL }}-${{ env.TIME }} | |
| # cache-matched-key is '' when cache/restore cannot hit cache even used "restore-keys" | |
| - uses: julia-actions/setup-julia@v1 | |
| if: ${{ steps.restore-cache.outputs.cache-matched-key == '' }} | |
| - uses: julia-actions/julia-buildpkg@v1 | |
| if: ${{ steps.restore-cache.outputs.cache-matched-key == '' }} | |
| - | |
| name: Install dependencies | |
| if: ${{ steps.restore-cache.outputs.cache-matched-key == '' }} | |
| run: | | |
| julia --project=benchmark benchmark/runbenchmarks.jl --cache-setup \ | |
| --arch=cpu \ | |
| --target=$TARGET_URL --baseline=$BASELINE_URL | |
| - | |
| uses: actions/cache/save@v3 | |
| if: ${{ steps.restore-cache.outputs.cache-matched-key == '' }} | |
| with: | |
| path: ~/.julia | |
| key: bm-${{ github.run_id }}-${{ github.run_attempt }}-${{ env.TARGET_URL }}-${{ env.BASELINE_URL }}-${{ env.TIME }} | |
| RunBenchmark: | |
| needs: [Cache] | |
| runs-on: ubuntu-latest | |
| env: | |
| TARGET_URL: ${{ github.event.inputs.target_url || format('{0}#{1}', github.event.pull_request.head.repo.html_url, github.event.pull_request.head.sha) }} | |
| BASELINE_URL: ${{ github.event.inputs.baseline_url || format('{0}#{1}', github.event.pull_request.base.repo.html_url, github.event.pull_request.base.sha) }} | |
| strategy: | |
| matrix: | |
| bm: [ | |
| # for test | |
| #"nnlib(attention,activations,gemm)", "nnlib(conv)", | |
| #"nnlib(pooling,softmax)", "nnlib(dropout,upsample)", | |
| "flux" | |
| ] | |
| steps: | |
| - | |
| name: Print target&baseline repo url | |
| run: | | |
| echo "Target repo url: $TARGET_URL" | |
| echo "Baseline repo url: $BASELINE_URL" | |
| - | |
| name: Checkout code | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - run: | | |
| git fetch origin benchmark-tuning:benchmark-tuning | |
| git fetch origin benchmark-results:benchmark-results | |
| - | |
| name: Setup julia | |
| uses: julia-actions/setup-julia@v1 | |
| - | |
| name: Restore cache | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.julia | |
| # This "key" is required, but the time of this job is definitely different from "Cache" job. | |
| # Therefore cache cannot be hit. So this is a fake key. | |
| key: bm-${{ github.run_id }}-${{ github.run_attempt }}-${{ env.TARGET_URL }}-${{ env.BASELINE_URL }}- | |
| restore-keys: | | |
| bm-${{ github.run_id }}-${{ github.run_attempt }}-${{ env.TARGET_URL }}-${{ env.BASELINE_URL }}- | |
| bm-${{ github.run_id }}-${{ github.run_attempt }}- | |
| - | |
| name: Build pkg | |
| uses: julia-actions/julia-buildpkg@v1 | |
| - | |
| id: benchmark | |
| name: Run benchmark | |
| run: | | |
| julia --project=benchmark benchmark/runbenchmarks.jl --pr \ | |
| --arch=cpu \ | |
| --target=$TARGET_URL --baseline=$BASELINE_URL \ | |
| --enable="${{ matrix.bm }}" \ | |
| --fetch-result | |
| - | |
| name: Print report | |
| run: | | |
| cat benchmark/report.md | |
| - | |
| name: Upload report | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: fluxml-benchmark-result-${{ matrix.bm }} | |
| path: | | |
| benchmark/result-baseline.json | |
| benchmark/result-target.json | |
| MergeReport: | |
| needs: [RunBenchmark] | |
| if: ${{ github.event_name == 'workflow_dispatch' }} | |
| runs-on: ubuntu-latest | |
| env: | |
| TARGET_URL: ${{ github.event.inputs.target_url || format('{0}#{1}', github.event.pull_request.head.repo.html_url, github.event.pull_request.head.sha) }} | |
| BASELINE_URL: ${{ github.event.inputs.baseline_url || format('{0}#{1}', github.event.pull_request.base.repo.html_url, github.event.pull_request.base.sha) }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - | |
| name: Download all reports | |
| uses: actions/download-artifact@v3 | |
| - uses: julia-actions/setup-julia@v1 | |
| - uses: julia-actions/julia-buildpkg@v1 | |
| - | |
| name: Merge reports | |
| run: | | |
| julia --project=benchmark benchmark/runbenchmarks.jl --merge-reports \ | |
| --arch=cpu \ | |
| --target=$TARGET_URL --baseline=$BASELINE_URL \ | |
| --push-result --push-password=${{ github.token }} | |
| - | |
| name: Upload final report | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: final-report | |
| path: benchmark/report.md | |
| Comment: | |
| needs: [MergeReport] | |
| if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.repository != '' && github.event.inputs.pr_id != '' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - | |
| name: Download report | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: final-report | |
| - | |
| name: Get app token | |
| id: get-app-token | |
| uses: tibdex/github-app-token@v1 | |
| with: | |
| app_id: ${{ secrets.BENCH_APP_ID }} | |
| installation_id: ${{ secrets.BENCH_INSTALL_ID }} | |
| private_key: ${{ secrets.BENCH_PRIVATE_KEY }} | |
| - | |
| name: Comment | |
| uses: peter-evans/create-or-update-comment@v2 | |
| with: | |
| repository: ${{ github.event.inputs.repository }} | |
| issue-number: ${{ github.event.inputs.pr_id }} | |
| token: ${{ steps.get-app-token.outputs.token }} | |
| body-file: report.md | |
| DryPrint: | |
| needs: [MergeReport] | |
| if: ${{ github.event.inputs.repository == '' || github.event.inputs.pr_id == '' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: final-report | |
| - run: | | |
| cat report.md | |
| GPU: | |
| if: ${{ github.event.inputs.enable_gpu && github.event.inputs.target_url != '' && github.event.inputs.baseline_url != '' }} | |
| runs-on: ubuntu-latest | |
| env: | |
| REPOSITORY: ${{ github.event.inputs.repository }} | |
| PR_ID: ${{ github.event.inputs.pr_id || github.event.pull_request.number }} | |
| TARGET_URL: ${{ github.event.inputs.target_url || format('{0}#{1}', github.event.pull_request.head.repo.html_url, github.event.pull_request.head.sha) }} | |
| BASELINE_URL: ${{ github.event.inputs.baseline_url || format('{0}#{1}', github.event.pull_request.base.repo.html_url, github.event.pull_request.base.sha) }} | |
| steps: | |
| - uses: buildkite/[email protected] | |
| with: | |
| buildkite-token: ${{ secrets.BENCH_BK_API_KEY }} | |
| pipeline: julialang/fluxmlbenchmarks-dot-jl | |
| branch: main | |
| message: ":github: Triggered from a GitHub Action" | |
| build-env-vars: '{ "TARGET_URL": "${{ env.TARGET_URL }}", "BASELINE_URL": "${{ env.BASELINE_URL}}", "REPOSITORY": "${{ env.REPOSITORY }}", "PR_ID": "${{ env.PR_ID }}" }' | |
| ignore-pipeline-branch-filter: true |