diff --git a/.github/workflows/performance_score_director.yml b/.github/workflows/performance_score_director.yml index 3bae1d2c..e190b246 100644 --- a/.github/workflows/performance_score_director.yml +++ b/.github/workflows/performance_score_director.yml @@ -73,7 +73,7 @@ jobs: echo "forks=15" > scoredirector-benchmark.properties echo "warmup_iterations=5" >> scoredirector-benchmark.properties echo "measurement_iterations=15" >> scoredirector-benchmark.properties - echo "relative_score_error_threshold=0.025" >> scoredirector-benchmark.properties + echo "relative_score_error_threshold=0.025C" >> scoredirector-benchmark.properties echo "score_director_type=cs" >> scoredirector-benchmark.properties echo "example=${{ matrix.example }}" >> scoredirector-benchmark.properties cat scoredirector-benchmark.properties @@ -86,12 +86,15 @@ jobs: - name: Phase 1 - Run the baseline configuration working-directory: ./timefold-solver-benchmarks + id: benchmark_baseline env: RUN_ID: ${{ github.event.inputs.baseline }} shell: bash run: | ./run-scoredirector.sh - + echo "RANGE_START=$(jq '.[0].primaryMetric.scoreConfidence[0]' ./timefold-solver-benchmarks/results/scoredirector/${{ github.event.inputs.baseline }}/results.json)" >> "$GITHUB_OUTPUT" + echo "RANGE_END=$(jq '.[0].primaryMetric.scoreConfidence[1]' ./timefold-solver-benchmarks/results/scoredirector/${{ github.event.inputs.baseline }}/results.json)" >> "$GITHUB_OUTPUT" + - name: Phase 2 - Checkout timefold-solver uses: actions/checkout@v4 with: @@ -134,16 +137,33 @@ jobs: run: mvn clean install -B -Dquickly -Dversion.tools.provider="${{ github.event.inputs.async_profiler_version }}" - name: Phase 2 - Run the benchmark on the new code + id: benchmark_new working-directory: ./timefold-solver-benchmarks env: RUN_ID: ${{ github.event.inputs.branch }} shell: bash run: | ./run-scoredirector.sh + echo "RANGE_START=$(jq '.[0].primaryMetric.scoreConfidence[0]' ./timefold-solver-benchmarks/results/scoredirector/${{ github.event.inputs.branch }}/results.json)" >> "$GITHUB_OUTPUT" + echo "RANGE_END=$(jq '.[0].primaryMetric.scoreConfidence[1]' ./timefold-solver-benchmarks/results/scoredirector/${{ github.event.inputs.branch }}/results.json)" >> "$GITHUB_OUTPUT" - name: Phase 3 - Archive benchmark data uses: actions/upload-artifact@v4 with: name: results-${{ matrix.example }}-${{ github.event.inputs.baseline }}_vs_${{ github.event.inputs.branch }} path: | - ./timefold-solver-benchmarks/results/scoredirector \ No newline at end of file + ./timefold-solver-benchmarks/results/scoredirector + + - name: Compare baseline with the branch + working-directory: ./timefold-solver-benchmarks + env: + OLD_RANGE_START: ${{ steps.benchmark_baseline.outputs.RANGE_START }} + OLD_RANGE_END: ${{ steps.benchmark_baseline.outputs.RANGE_END }} + NEW_RANGE_START: ${{ steps.benchmark_new.outputs.RANGE_START }} + NEW_RANGE_END: ${{ steps.benchmark_new.outputs.RANGE_END }} + shell: bash + run: | + echo "OLD_RANGE_START=$OLD_RANGE_START" + echo "OLD_RANGE_END=$OLD_RANGE_END" + echo "NEW_RANGE_START=$NEW_RANGE_START" + echo "NEW_RANGE_END=$NEW_RANGE_END" \ No newline at end of file