Performance Regression Test - Score Director #36
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# - Runs entirely on a single machine. | |
# - The baseline is established first, then the branch under test is measured. | |
# - Both runs fail if the benchmark error is over predefined thresholds. | |
# - Then, if both are below thresholds and neither failed, those results must be directly comparable. | |
# - Therefore, if the difference between the two is over the threshold, then the branch is considered to have regressed. | |
name: Performance Regression Test - Score Director | |
on: | |
workflow_dispatch: | |
inputs: | |
jdk: | |
description: 'JDK version' | |
default: '21' | |
required: true | |
baseline: | |
description: 'Timefold Solver release' | |
default: '1.14.0' | |
required: true | |
branch: | |
description: 'Branch to benchmark (needs to use 999-SNAPSHOT)' | |
default: 'main' | |
required: true | |
branch_owner: | |
description: 'User owning the branch' | |
default: 'TimefoldAI' | |
required: true | |
async_profiler_version: | |
description: 'async-profiler version' | |
default: '3.0' | |
required: true | |
jobs: | |
test: | |
runs-on: ubuntu-latest | |
strategy: | |
fail-fast: false # Jobs fail if the benchmark error is over predefined thresholds; other benchmarks continue. | |
matrix: | |
example: [cloud_balancing, conference_scheduling, curriculum_course, examination, machine_reassignment, meeting_scheduling, nurse_rostering, patient_admission_scheduling, task_assigning, traveling_tournament, tsp, vehicle_routing] | |
env: | |
MVN_USERNAME: '${{ secrets.JFROG_ENTERPRISE_READ_ONLY_ACCESS_USERNAME }}' | |
MVN_PASSWORD: '${{ secrets.JFROG_ENTERPRISE_READ_ONLY_ACCESS_TOKEN }}' | |
steps: | |
- name: Phase 0 - Checkout timefold-solver-benchmarks | |
uses: actions/checkout@v4 | |
with: | |
repository: TimefoldAI/timefold-solver-benchmarks | |
path: ./timefold-solver-benchmarks | |
- name: Phase 0 - Setup JDK and Maven | |
uses: actions/setup-java@v4 | |
with: | |
java-version: ${{ github.event.inputs.jdk }} | |
distribution: 'temurin' | |
cache: 'maven' | |
server-id: 'timefold-solver-enterprise' | |
server-username: 'MVN_USERNAME' | |
server-password: 'MVN_PASSWORD' | |
- name: Phase 0 - Setup Async Profiler | |
working-directory: ./timefold-solver-benchmarks | |
run: | | |
export FILENAME=async-profiler-${{ github.event.inputs.async_profiler_version }}-linux-x64.tar.gz | |
wget https://github.com/async-profiler/async-profiler/releases/download/v${{ github.event.inputs.async_profiler_version }}/$FILENAME | |
tar -xzf $FILENAME | |
ls -l | |
# Fine-tuned for stability on GHA. | |
- name: Phase 0 - Configure the benchmark | |
working-directory: ./timefold-solver-benchmarks | |
shell: bash | |
run: | | |
echo "forks=15" > scoredirector-benchmark.properties | |
echo "warmup_iterations=5" >> scoredirector-benchmark.properties | |
echo "measurement_iterations=15" >> scoredirector-benchmark.properties | |
echo "score_director_type=cs" >> scoredirector-benchmark.properties | |
echo "example=${{ matrix.example }}" >> scoredirector-benchmark.properties | |
cat scoredirector-benchmark.properties | |
chmod +x run-scoredirector.sh | |
- name: Phase 1 - Compile the benchmark | |
working-directory: ./timefold-solver-benchmarks | |
shell: bash | |
run: mvn clean install -B -Dquickly -Dversion.ai.timefold.solver=${{ github.event.inputs.baseline }} -Dversion.tools.provider="${{ github.event.inputs.async_profiler_version }}" | |
- name: Phase 1 - Run the baseline configuration | |
working-directory: ./timefold-solver-benchmarks | |
env: | |
RUN_ID: ${{ github.event.inputs.baseline }} | |
shell: bash | |
run: | | |
./run-scoredirector.sh | |
- name: Phase 2 - Checkout timefold-solver | |
uses: actions/checkout@v4 | |
with: | |
repository: ${{ github.event.inputs.branch_owner }}/timefold-solver | |
ref: ${{ github.event.inputs.branch }} | |
path: ./timefold-solver | |
- name: Phase 2 - Quickly build timefold-solver | |
working-directory: ./timefold-solver | |
shell: bash | |
run: mvn -B -Dquickly clean install | |
# Clone timefold-solver-enterprise | |
- name: Phase 2 - Checkout timefold-solver-enterprise (PR) # Checkout the PR branch first, if it exists | |
id: checkout-solver-enterprise | |
uses: actions/checkout@v4 | |
continue-on-error: true | |
with: | |
repository: TimefoldAI/timefold-solver-enterprise | |
ref: ${{ github.event.inputs.branch }} | |
token: ${{ secrets.BENCHMARK_PUBLISH_TOKEN }} | |
path: ./timefold-solver-enterprise | |
- name: Phase 2 - Checkout timefold-solver-enterprise (main) # Checkout the main branch if the PR branch does not exist | |
if: steps.checkout-solver-enterprise.outcome != 'success' | |
uses: actions/checkout@v4 | |
with: | |
repository: TimefoldAI/timefold-solver-enterprise | |
ref: main | |
token: ${{ secrets.BENCHMARK_PUBLISH_TOKEN }} | |
path: ./timefold-solver-enterprise | |
- name: Phase 2 - Quickly build timefold-solver-enterprise | |
working-directory: ./timefold-solver-enterprise | |
shell: bash | |
run: mvn -B -Dquickly clean install | |
- name: Phase 2 - Compile the benchmarks | |
working-directory: ./timefold-solver-benchmarks | |
shell: bash | |
run: mvn clean install -B -Dquickly -Dversion.tools.provider="${{ github.event.inputs.async_profiler_version }}" | |
- name: Phase 2 - Run the benchmark on the new code | |
working-directory: ./timefold-solver-benchmarks | |
env: | |
RUN_ID: ${{ github.event.inputs.branch }} | |
shell: bash | |
run: | | |
./run-scoredirector.sh | |
- name: Phase 3 - Archive benchmark data | |
uses: actions/upload-artifact@v4 | |
with: | |
name: results-${{ matrix.example }}-${{ github.event.inputs.baseline }}_vs_${{ github.event.inputs.branch }} | |
path: | | |
./timefold-solver-benchmarks/results/scoredirector |