Outputs

TimefoldAI · Oct 2, 2024 · 7b388df · 7b388df
1 parent a8024f6
commit 7b388df
Showing 1 changed file with 23 additions and 3 deletions.
diff --git a/.github/workflows/performance_score_director.yml b/.github/workflows/performance_score_director.yml
@@ -73,7 +73,7 @@ jobs:
           echo "forks=15" > scoredirector-benchmark.properties
           echo "warmup_iterations=5" >> scoredirector-benchmark.properties
           echo "measurement_iterations=15" >> scoredirector-benchmark.properties
-          echo "relative_score_error_threshold=0.025" >> scoredirector-benchmark.properties
+          echo "relative_score_error_threshold=0.025C" >> scoredirector-benchmark.properties
           echo "score_director_type=cs" >> scoredirector-benchmark.properties
           echo "example=${{ matrix.example }}" >> scoredirector-benchmark.properties
           cat scoredirector-benchmark.properties
@@ -86,12 +86,15 @@ jobs:
 
       - name: Phase 1 - Run the baseline configuration
         working-directory: ./timefold-solver-benchmarks
+        id: benchmark_baseline
         env:
           RUN_ID: ${{ github.event.inputs.baseline }}
         shell: bash
         run: |
           ./run-scoredirector.sh
-          
+          echo "RANGE_START=$(jq '.[0].primaryMetric.scoreConfidence[0]' ./timefold-solver-benchmarks/results/scoredirector/${{ github.event.inputs.baseline }}/results.json)" >> "$GITHUB_OUTPUT"
+          echo "RANGE_END=$(jq '.[0].primaryMetric.scoreConfidence[1]' ./timefold-solver-benchmarks/results/scoredirector/${{ github.event.inputs.baseline }}/results.json)" >> "$GITHUB_OUTPUT"
+
       - name: Phase 2 - Checkout timefold-solver
         uses: actions/checkout@v4
         with:
@@ -134,16 +137,33 @@ jobs:
         run: mvn clean install -B -Dquickly -Dversion.tools.provider="${{ github.event.inputs.async_profiler_version }}"
 
       - name: Phase 2 - Run the benchmark on the new code
+        id: benchmark_new
         working-directory: ./timefold-solver-benchmarks
         env:
           RUN_ID: ${{ github.event.inputs.branch }}
         shell: bash
         run: |
           ./run-scoredirector.sh
+          echo "RANGE_START=$(jq '.[0].primaryMetric.scoreConfidence[0]' ./timefold-solver-benchmarks/results/scoredirector/${{ github.event.inputs.branch }}/results.json)" >> "$GITHUB_OUTPUT"
+          echo "RANGE_END=$(jq '.[0].primaryMetric.scoreConfidence[1]' ./timefold-solver-benchmarks/results/scoredirector/${{ github.event.inputs.branch }}/results.json)" >> "$GITHUB_OUTPUT"
 
       - name: Phase 3 - Archive benchmark data
         uses: actions/upload-artifact@v4
         with:
           name: results-${{ matrix.example }}-${{ github.event.inputs.baseline }}_vs_${{ github.event.inputs.branch }}
           path: |
-            ./timefold-solver-benchmarks/results/scoredirector
+            ./timefold-solver-benchmarks/results/scoredirector
+
+      - name: Compare baseline with the branch
+        working-directory: ./timefold-solver-benchmarks
+        env:
+          OLD_RANGE_START: ${{ steps.benchmark_baseline.outputs.RANGE_START }}
+          OLD_RANGE_END:   ${{ steps.benchmark_baseline.outputs.RANGE_END }}
+          NEW_RANGE_START: ${{ steps.benchmark_new.outputs.RANGE_START }}
+          NEW_RANGE_END:   ${{ steps.benchmark_new.outputs.RANGE_END }}
+        shell: bash
+        run: |
+          echo "OLD_RANGE_START=$OLD_RANGE_START"
+          echo "OLD_RANGE_END=$OLD_RANGE_END"
+          echo "NEW_RANGE_START=$NEW_RANGE_START"
+          echo "NEW_RANGE_END=$NEW_RANGE_END"