From 832824dc48e395c358f54f016d4a16eebb25e4ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Thu, 9 Mar 2023 23:15:38 +0200 Subject: [PATCH 001/111] refactor(ci): Refactor the workflow to use self-hosted runners and upload the build files as artifacts. --- .../{cmake.yml => build-and-test.yml} | 39 +++++++++++-------- 1 file changed, 22 insertions(+), 17 deletions(-) rename .github/workflows/{cmake.yml => build-and-test.yml} (78%) diff --git a/.github/workflows/cmake.yml b/.github/workflows/build-and-test.yml similarity index 78% rename from .github/workflows/cmake.yml rename to .github/workflows/build-and-test.yml index 4334942b..3b8e4572 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/build-and-test.yml @@ -5,18 +5,16 @@ on: [push] env: # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) BUILD_TYPE: Release + JOB_COUNT: 3 jobs: build: - # The CMake configure and build commands are platform agnostic and should work equally - # well on Windows or Mac. You can convert this to a matrix build if you need - # cross-platform coverage. - # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix - runs-on: ubuntu-latest + name: Build SHOT + runs-on: [self-hosted, linux] steps: - uses: actions/checkout@v2 - + - name: Checkout submodules run: git submodule update --init --recursive @@ -24,7 +22,7 @@ jobs: # Some projects don't allow in-source building, so create a separate build directory # We'll use this as our working directory for all subsequent commands run: cmake -E make_directory ${{runner.workspace}}/build - + - name: Install dependencies shell: bash run: sudo apt-get update ; sudo apt-get install libc6-dev libbz2-dev zlib1g-dev liblapack-dev libnauty2-dev libopenblas-base libopenblas-dev libmumps-dev @@ -35,7 +33,7 @@ jobs: with: path: ${{runner.workspace}}/ThirdParty/Cbc key: ${{ runner.os }}-cbc - + - name: Cache Ipopt id: cache-ipopt uses: actions/cache@v2 @@ -43,32 +41,31 @@ jobs: path: ${{runner.workspace}}/ThirdParty/Ipopt key: ${{ runner.os }}-ipopt - - name: Download Coinbrew + - name: Setup Coinbrew if: steps.cache-ipopt.outputs.cache-hit != 'true' || steps.cache-cbc.outputs.cache-hit != 'true' - shell: bash working-directory: ${{runner.workspace}}/build - run: git clone https://github.com/coin-or/coinbrew + run: git -C "coinbrew" pull || git clone https://github.com/coin-or/coinbrew - name: Download and build Ipopt if: steps.cache-ipopt.outputs.cache-hit != 'true' shell: bash working-directory: ${{runner.workspace}}/build/coinbrew - run: ./coinbrew build Ipopt --verbosity 2 --prefix=${{runner.workspace}}/ThirdParty/Ipopt --no-prompt --tests none + run: ./coinbrew build Ipopt --verbosity 2 --prefix=${{runner.workspace}}/ThirdParty/Ipopt -j $JOB_COUNT --no-prompt --tests none - name: Download and build Cbc if: steps.cache-cbc.outputs.cache-hit != 'true' shell: bash working-directory: ${{runner.workspace}}/build/coinbrew - run: rm -rf build dist ; ./coinbrew build Cbc@stable/2.10 --verbosity 2 --prefix=${{runner.workspace}}/ThirdParty/Cbc --no-prompt --tests none --no-third-party + run: rm -rf build dist ; ./coinbrew build Cbc@stable/2.10 --verbosity 2 -j $JOB_COUNT --prefix=${{runner.workspace}}/ThirdParty/Cbc --no-prompt --tests none --no-third-party - name: Configure CMake # Use a bash shell so we can use the same syntax for environment variable # access regardless of the host operating system shell: bash working-directory: ${{runner.workspace}}/build - # Note the current convention is to use the -S and -B options here to specify source - # and build directories, but this is only available with CMake 3.13 and higher. + # Note the current convention is to use the -S and -B options here to specify source + # and build directories, but this is only available with CMake 3.13 and higher. # The CMake binaries on the Github Actions machines are (as of this writing) 3.12 run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DHAS_CBC=on -DCBC_DIR=${{runner.workspace}}/ThirdParty/Cbc -DHAS_IPOPT=ON -DIPOPT_DIR=${{runner.workspace}}/ThirdParty/Ipopt -DHAS_GAMS=off -DHAS_CPLEX=off -DHAS_GUROBI=off -DCOMPILE_TESTS=on -DSPDLOG_STATIC=on @@ -76,11 +73,19 @@ jobs: working-directory: ${{runner.workspace}}/build shell: bash # Execute the build. You can specify a specific target with "--target " - run: cmake --build . --config $BUILD_TYPE + run: cmake --build . --config $BUILD_TYPE -j $JOB_COUNT - name: Test working-directory: ${{runner.workspace}}/build shell: bash - # Execute tests defined by the CMake configuration. + # Execute tests defined by the CMake configuration. # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail run: ctest -C $BUILD_TYPE --output-on-failure + + # Upload all the build files as artifacts. + - uses: actions/upload-artifact@v3 + with: + name: shot-build + path: ${{runner.workspace}}/build + retention-days: 4 + From d004c65ebe1d7e20e6a128bf16ad4a32daf0f917 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Thu, 9 Mar 2023 23:17:05 +0200 Subject: [PATCH 002/111] fix(ci): Auto accept the dependency installation --- .github/workflows/build-and-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 3b8e4572..d2697d87 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -25,7 +25,7 @@ jobs: - name: Install dependencies shell: bash - run: sudo apt-get update ; sudo apt-get install libc6-dev libbz2-dev zlib1g-dev liblapack-dev libnauty2-dev libopenblas-base libopenblas-dev libmumps-dev + run: sudo apt-get update ; sudo apt-get install -y libc6-dev libbz2-dev zlib1g-dev liblapack-dev libnauty2-dev libopenblas-base libopenblas-dev libmumps-dev - name: Cache Cbc id: cache-cbc From d2b3df427058daa235aba19d6c7fe42184ee5c10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Thu, 9 Mar 2023 23:19:47 +0200 Subject: [PATCH 003/111] fix(ci): Update cache version and install pkgconfig package --- .github/workflows/build-and-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index d2697d87..8e084128 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -25,18 +25,18 @@ jobs: - name: Install dependencies shell: bash - run: sudo apt-get update ; sudo apt-get install -y libc6-dev libbz2-dev zlib1g-dev liblapack-dev libnauty2-dev libopenblas-base libopenblas-dev libmumps-dev + run: sudo apt-get update ; sudo apt-get install -y libc6-dev libbz2-dev zlib1g-dev liblapack-dev libnauty2-dev libopenblas-base libopenblas-dev libmumps-dev pkgconf - name: Cache Cbc id: cache-cbc - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ${{runner.workspace}}/ThirdParty/Cbc key: ${{ runner.os }}-cbc - name: Cache Ipopt id: cache-ipopt - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ${{runner.workspace}}/ThirdParty/Ipopt key: ${{ runner.os }}-ipopt From 9644bfca8f49c06f8a7d1037ba863ffdc3beee50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Thu, 9 Mar 2023 23:56:47 +0200 Subject: [PATCH 004/111] fix(ci): Temporarily remove the build artifact upload --- .github/workflows/build-and-test.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 8e084128..ed6c6610 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -82,10 +82,3 @@ jobs: # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail run: ctest -C $BUILD_TYPE --output-on-failure - # Upload all the build files as artifacts. - - uses: actions/upload-artifact@v3 - with: - name: shot-build - path: ${{runner.workspace}}/build - retention-days: 4 - From 4b1fcbddce68f557fafea00a33757ece172bab3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Tue, 14 Mar 2023 18:53:10 +0200 Subject: [PATCH 005/111] refactor(ci): Refactor the build process to use reusable workflows and readded the artifact storage. --- .github/workflows/build-and-test.yml | 84 ------------ .github/workflows/build-push.yml | 11 ++ .github/workflows/build-workflow.yml | 195 +++++++++++++++++++++++++++ 3 files changed, 206 insertions(+), 84 deletions(-) delete mode 100644 .github/workflows/build-and-test.yml create mode 100644 .github/workflows/build-push.yml create mode 100644 .github/workflows/build-workflow.yml diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml deleted file mode 100644 index ed6c6610..00000000 --- a/.github/workflows/build-and-test.yml +++ /dev/null @@ -1,84 +0,0 @@ -name: CMake - -on: [push] - -env: - # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) - BUILD_TYPE: Release - JOB_COUNT: 3 - -jobs: - build: - name: Build SHOT - runs-on: [self-hosted, linux] - - steps: - - uses: actions/checkout@v2 - - - name: Checkout submodules - run: git submodule update --init --recursive - - - name: Create Build Environment - # Some projects don't allow in-source building, so create a separate build directory - # We'll use this as our working directory for all subsequent commands - run: cmake -E make_directory ${{runner.workspace}}/build - - - name: Install dependencies - shell: bash - run: sudo apt-get update ; sudo apt-get install -y libc6-dev libbz2-dev zlib1g-dev liblapack-dev libnauty2-dev libopenblas-base libopenblas-dev libmumps-dev pkgconf - - - name: Cache Cbc - id: cache-cbc - uses: actions/cache@v3 - with: - path: ${{runner.workspace}}/ThirdParty/Cbc - key: ${{ runner.os }}-cbc - - - name: Cache Ipopt - id: cache-ipopt - uses: actions/cache@v3 - with: - path: ${{runner.workspace}}/ThirdParty/Ipopt - key: ${{ runner.os }}-ipopt - - - name: Setup Coinbrew - if: steps.cache-ipopt.outputs.cache-hit != 'true' || steps.cache-cbc.outputs.cache-hit != 'true' - shell: bash - working-directory: ${{runner.workspace}}/build - run: git -C "coinbrew" pull || git clone https://github.com/coin-or/coinbrew - - - name: Download and build Ipopt - if: steps.cache-ipopt.outputs.cache-hit != 'true' - shell: bash - working-directory: ${{runner.workspace}}/build/coinbrew - run: ./coinbrew build Ipopt --verbosity 2 --prefix=${{runner.workspace}}/ThirdParty/Ipopt -j $JOB_COUNT --no-prompt --tests none - - - name: Download and build Cbc - if: steps.cache-cbc.outputs.cache-hit != 'true' - shell: bash - working-directory: ${{runner.workspace}}/build/coinbrew - run: rm -rf build dist ; ./coinbrew build Cbc@stable/2.10 --verbosity 2 -j $JOB_COUNT --prefix=${{runner.workspace}}/ThirdParty/Cbc --no-prompt --tests none --no-third-party - - - name: Configure CMake - # Use a bash shell so we can use the same syntax for environment variable - # access regardless of the host operating system - shell: bash - working-directory: ${{runner.workspace}}/build - # Note the current convention is to use the -S and -B options here to specify source - # and build directories, but this is only available with CMake 3.13 and higher. - # The CMake binaries on the Github Actions machines are (as of this writing) 3.12 - run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DHAS_CBC=on -DCBC_DIR=${{runner.workspace}}/ThirdParty/Cbc -DHAS_IPOPT=ON -DIPOPT_DIR=${{runner.workspace}}/ThirdParty/Ipopt -DHAS_GAMS=off -DHAS_CPLEX=off -DHAS_GUROBI=off -DCOMPILE_TESTS=on -DSPDLOG_STATIC=on - - - name: Build - working-directory: ${{runner.workspace}}/build - shell: bash - # Execute the build. You can specify a specific target with "--target " - run: cmake --build . --config $BUILD_TYPE -j $JOB_COUNT - - - name: Test - working-directory: ${{runner.workspace}}/build - shell: bash - # Execute tests defined by the CMake configuration. - # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail - run: ctest -C $BUILD_TYPE --output-on-failure - diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml new file mode 100644 index 00000000..4685069c --- /dev/null +++ b/.github/workflows/build-push.yml @@ -0,0 +1,11 @@ +name: Build + +on: [push] + +jobs: + build: + uses: ./.github/workflows/build-workflow.yml + with: + job_count: 3 + build_type: Release + retention: 30 diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml new file mode 100644 index 00000000..c61378bd --- /dev/null +++ b/.github/workflows/build-workflow.yml @@ -0,0 +1,195 @@ +name: Build and Test + +on: + workflow_call: + inputs: + build_type: + description: 'The CMake build type, can be either: Release, Debug, RelWithDebInfo, etc.' + default: Release + required: false + type: string + job_count: + description: 'The amount of jobs to use when building the program and dependencies.' + default: 1 + required: false + type: number + # Map the workflow outputs to job outputs + version: + description: 'The version to build SHOT as.' + default: ${{ github.sha }} + required: false + type: string + retention: + description: 'The artifact retention period (in days)' + default: 60 + required: false + type: number + outputs: + test_artifact: + description: "Test artifact ID" + value: ${{ jobs.build.outputs.test_artifact }} + options_artifact: + description: "SHOT Optionsfile artifact ID" + value: ${{ jobs.build.outputs.options_artifact }} + binary_artifact: + description: "SHOT Binary artifact ID" + value: ${{ jobs.build.outputs.binary_artifact }} + package_artifact: + description: "SHOT Package artifact ID" + value: ${{ jobs.build.outputs.package_artifact }} + library_artifact: + description: "SHOT Libraries artifact ID" + value: ${{ jobs.build.outputs.library_artifact }} + + +env: + # Customize + BUILD_TYPE: ${{ inputs.build_type }} + JOB_COUNT: ${{ inputs.job_count }} + VERSION: ${{ inputs.version }} + +jobs: + build: + outputs: + test_artifact: ${{ steps.artifact_outputs.test }} + options_artifact: ${{ steps.artifact_outputs.options }} + binary_artifact: ${{ steps.artifact_outputs.binary }} + package_artifact: ${{ steps.artifact_outputs.package }} + library_artifact: ${{ steps.artifact_outputs.library }} + name: Build SHOT + runs-on: [self-hosted, linux] + + steps: + - uses: actions/checkout@v2 + + - name: Checkout submodules + run: git submodule update --init --recursive + + - name: Create Build Environment + # Some projects don't allow in-source building, so create a separate build directory + # We'll use this as our working directory for all subsequent commands + run: cmake -E make_directory ${{runner.workspace}}/build + + - name: Install dependencies + shell: bash + run: sudo apt-get update ; sudo apt-get install -y libc6-dev libbz2-dev zlib1g-dev liblapack-dev libnauty2-dev libopenblas-base libopenblas-dev libmumps-dev pkgconf + + - name: Cache Cbc + id: cache-cbc + uses: actions/cache@v3 + with: + path: ${{runner.workspace}}/ThirdParty/Cbc + key: ${{ runner.os }}-cbc + + - name: Cache Ipopt + id: cache-ipopt + uses: actions/cache@v3 + with: + path: ${{runner.workspace}}/ThirdParty/Ipopt + key: ${{ runner.os }}-ipopt + + - name: Setup Coinbrew + if: steps.cache-ipopt.outputs.cache-hit != 'true' || steps.cache-cbc.outputs.cache-hit != 'true' + shell: bash + working-directory: ${{runner.workspace}}/build + run: git -C "coinbrew" pull || git clone https://github.com/coin-or/coinbrew + + - name: Download and build Ipopt + if: steps.cache-ipopt.outputs.cache-hit != 'true' + shell: bash + working-directory: ${{runner.workspace}}/build/coinbrew + run: ./coinbrew build Ipopt --verbosity 2 --prefix=${{runner.workspace}}/ThirdParty/Ipopt -j $JOB_COUNT --no-prompt --tests none + + - name: Download and build Cbc + if: steps.cache-cbc.outputs.cache-hit != 'true' + shell: bash + working-directory: ${{runner.workspace}}/build/coinbrew + run: rm -rf build dist ; ./coinbrew build Cbc@stable/2.10 --verbosity 2 -j $JOB_COUNT --prefix=${{runner.workspace}}/ThirdParty/Cbc --no-prompt --tests none --no-third-party + + - name: Configure CMake + # Use a bash shell so we can use the same syntax for environment variable + # access regardless of the host operating system + shell: bash + working-directory: ${{runner.workspace}}/build + # Note the current convention is to use the -S and -B options here to specify source + # and build directories, but this is only available with CMake 3.13 and higher. + # The CMake binaries on the Github Actions machines are (as of this writing) 3.12 + run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DHAS_CBC=on -DCBC_DIR=${{runner.workspace}}/ThirdParty/Cbc -DHAS_IPOPT=ON -DIPOPT_DIR=${{runner.workspace}}/ThirdParty/Ipopt -DHAS_GAMS=off -DHAS_CPLEX=off -DHAS_GUROBI=off -DCOMPILE_TESTS=on -DSPDLOG_STATIC=on + + - name: Build + working-directory: ${{runner.workspace}}/build + shell: bash + # Execute the build. You can specify a specific target with "--target " + run: cmake --build . --config $BUILD_TYPE -j $JOB_COUNT + + - name: Generate Optionsfile + working-directory: ${{runner.workspace}}/build + shell: bash + run: ./SHOT --opt + + - name: Test + id: test + working-directory: ${{runner.workspace}}/build + shell: bash + # Execute tests defined by the CMake configuration. + # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail + run: ctest -C $BUILD_TYPE --output-on-failure --output-junit test-output/ctest-junit-output.xml --output-log test-output/ctest-test.log + + - name: Construct package + working-directory: ${{runner.workspace}}/build + shell: bash + # Execute tests defined by the CMake configuration. + # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail + run: make package + + # This will always run as we always want the test output, however if the files are not found, we just warn the user. + - uses: actions/upload-artifact@v3 + if: ${{ always() }} + name: Upload test results + with: + name: test-logs-${{ env.VERSION }} + path: | + build/test-output/*.log + build/test-output/*.xml + retention-days: ${{ inputs.version }} + + - uses: actions/upload-artifact@v3 + name: Upload SHOT Optionsfile + with: + name: shot-optsfile-${{ env.VERSION }} + path: build/SHOT.opt + retention-days: ${{ inputs.version }} + + - uses: actions/upload-artifact@v3 + name: Upload SHOT File + with: + name: shot-binary-${{ env.VERSION }} + path: build/SHOT + retention-days: ${{ inputs.version }} + + - uses: actions/upload-artifact@v3 + name: Upload SHOT File + with: + name: shot-package-${{ env.VERSION }} + path: build/SHOT-*-Linux.zip + retention-days: ${{ inputs.version }} + + - uses: actions/upload-artifact@v3 + name: Upload SHOT Libraries + with: + name: shot-libraries-${{ env.VERSION }} + path: | + build/*.so + build/*.a + retention-days: ${{ inputs.version }} + + - name: Output artifact names + shell: bash + # Execute tests defined by the CMake configuration. + # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail + run: | + echo "test=test-logs-${{ env.VERSION }}" >> $GITHUB_OUTPUT + echo "options=shot-optsfile-${{ env.VERSION }}" >> $GITHUB_OUTPUT + echo "binary=shot-binary-${{ env.VERSION }}" >> $GITHUB_OUTPUT + echo "package=shot-package-${{ env.VERSION }}" >> $GITHUB_OUTPUT + echo "library=shot-libraries-${{ env.VERSION }}" >> $GITHUB_OUTPUT From b96269bb69b7a9bff8340766bdf924e51869213b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Tue, 14 Mar 2023 19:05:14 +0200 Subject: [PATCH 006/111] fix(ci): Attempt to use the runner workspace as base for the artifact outputs --- .github/workflows/build-workflow.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index c61378bd..f8f769dd 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -149,29 +149,29 @@ jobs: with: name: test-logs-${{ env.VERSION }} path: | - build/test-output/*.log - build/test-output/*.xml + ${{runner.workspace}}/build/test-output/*.log + ${{runner.workspace}}/build/test-output/*.xml retention-days: ${{ inputs.version }} - uses: actions/upload-artifact@v3 name: Upload SHOT Optionsfile with: name: shot-optsfile-${{ env.VERSION }} - path: build/SHOT.opt + path: ${{runner.workspace}}/build/SHOT.opt retention-days: ${{ inputs.version }} - uses: actions/upload-artifact@v3 name: Upload SHOT File with: name: shot-binary-${{ env.VERSION }} - path: build/SHOT + path: ${{runner.workspace}}/build/SHOT retention-days: ${{ inputs.version }} - uses: actions/upload-artifact@v3 name: Upload SHOT File with: name: shot-package-${{ env.VERSION }} - path: build/SHOT-*-Linux.zip + path: ${{runner.workspace}}/build/SHOT-*-Linux.zip retention-days: ${{ inputs.version }} - uses: actions/upload-artifact@v3 @@ -179,8 +179,8 @@ jobs: with: name: shot-libraries-${{ env.VERSION }} path: | - build/*.so - build/*.a + ${{runner.workspace}}/build/*.so + ${{runner.workspace}}/build/*.a retention-days: ${{ inputs.version }} - name: Output artifact names From f8780d60e1d708966e1dce71c7c856a3ce190c16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Tue, 14 Mar 2023 21:06:54 +0200 Subject: [PATCH 007/111] fix(ci): Use correct input for the retention --- .github/workflows/build-workflow.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index f8f769dd..fc61d88b 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -151,28 +151,28 @@ jobs: path: | ${{runner.workspace}}/build/test-output/*.log ${{runner.workspace}}/build/test-output/*.xml - retention-days: ${{ inputs.version }} + retention-days: ${{ inputs.retention }} - uses: actions/upload-artifact@v3 name: Upload SHOT Optionsfile with: name: shot-optsfile-${{ env.VERSION }} path: ${{runner.workspace}}/build/SHOT.opt - retention-days: ${{ inputs.version }} + retention-days: ${{ inputs.retention }} - uses: actions/upload-artifact@v3 name: Upload SHOT File with: name: shot-binary-${{ env.VERSION }} path: ${{runner.workspace}}/build/SHOT - retention-days: ${{ inputs.version }} + retention-days: ${{ inputs.retention }} - uses: actions/upload-artifact@v3 name: Upload SHOT File with: name: shot-package-${{ env.VERSION }} path: ${{runner.workspace}}/build/SHOT-*-Linux.zip - retention-days: ${{ inputs.version }} + retention-days: ${{ inputs.retention }} - uses: actions/upload-artifact@v3 name: Upload SHOT Libraries @@ -181,7 +181,7 @@ jobs: path: | ${{runner.workspace}}/build/*.so ${{runner.workspace}}/build/*.a - retention-days: ${{ inputs.version }} + retention-days: ${{ inputs.retention }} - name: Output artifact names shell: bash From ee1720ff8d97a6cad0dbbf5374d6c072f00f8040 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Tue, 14 Mar 2023 23:18:35 +0200 Subject: [PATCH 008/111] fix(ci): Update checkout version and fix naming for one step --- .github/workflows/build-workflow.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index fc61d88b..4cd8582b 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -60,7 +60,7 @@ jobs: runs-on: [self-hosted, linux] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Checkout submodules run: git submodule update --init --recursive @@ -168,7 +168,7 @@ jobs: retention-days: ${{ inputs.retention }} - uses: actions/upload-artifact@v3 - name: Upload SHOT File + name: Upload SHOT Package file with: name: shot-package-${{ env.VERSION }} path: ${{runner.workspace}}/build/SHOT-*-Linux.zip From 087496ba4e45d0489377f3192fb22a823e6a8340 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Fri, 17 Mar 2023 13:06:22 +0200 Subject: [PATCH 009/111] feat(ci): Added a step to publish the test results. --- .github/workflows/build-push.yml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 4685069c..47cebe0b 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -3,9 +3,27 @@ name: Build on: [push] jobs: - build: + build-test: uses: ./.github/workflows/build-workflow.yml with: job_count: 3 build_type: Release retention: 30 + + publish-test: + needs: build-test + runs-on: [self-hosted, linux] + steps: + - uses: actions/download-artifact@v3 + name: Download test results + with: + name: ${needs.build-test.outputs.test_artifact} + + - name: Publish Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + files: | + test-output/*.xml + + From cb8445ea17a4bcf2a2a369af0d89b820fa83e3b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Fri, 17 Mar 2023 14:12:31 +0200 Subject: [PATCH 010/111] feat(ci): Added runner-init.yml script to initialize runners and updated the build script to use docker-runners --- .github/workflows/build-push.yml | 4 +- misc/runner-init.yml | 68 ++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 misc/runner-init.yml diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 47cebe0b..bd5e12d8 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -4,6 +4,7 @@ on: [push] jobs: build-test: + name: Build & Test uses: ./.github/workflows/build-workflow.yml with: job_count: 3 @@ -12,7 +13,8 @@ jobs: publish-test: needs: build-test - runs-on: [self-hosted, linux] + name: Publish test results + runs-on: [self-hosted, docker] steps: - uses: actions/download-artifact@v3 name: Download test results diff --git a/misc/runner-init.yml b/misc/runner-init.yml new file mode 100644 index 00000000..f23f1e56 --- /dev/null +++ b/misc/runner-init.yml @@ -0,0 +1,68 @@ +#cloud-config: +package_update: true +package_upgrade: true + +packages: + # Required for GitHub runner + - zlib1g + - libssl1.1 + - libkrb5-3 + - liblttng-ust-ctl4 + - liblttng-ust0 + - build-essential + - jq + - libssl-dev + - ca-certificates + - curl + - gnupg + - lsb-release + +users: + - default + - name: ghrunner + groups: sudo + shell: /bin/bash + sudo: ['ALL=(ALL) NOPASSWD:ALL'] + +runcmd: + # Install docker + - mkdir -m 0755 -p /etc/apt/keyrings + - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg + - echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + - chmod a+r /etc/apt/keyrings/docker.gpg + - apt-get update + - apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + # Install CMake + - 'curl -O -L https://github.com/Kitware/CMake/releases/download/v3.26.0/cmake-3.26.0-linux-x86_64.sh' + - mv cmake-3.26.0-linux-x86_64.sh /opt/ + - cd /opt/ + - chmod +x cmake-3.26.0-linux-x86_64.sh + - bash ./cmake-3.26.0-linux-x86_64.sh --skip-license + - ln -s /opt/bin/* /usr/local/bin + - echo "Installed CMake" + - cd /home/ghrunner + # Create the GH Runner directory. + - su 'ghrunner' -c "mkdir actions-runner" + - echo "Directory created" + - cd actions-runner + - echo "Swapped directory" + # We fetch a temporary runner creation token to allow adding the runner to the repo see: https://developer.github.com/v3/actions/self-hosted-runners/ for more information. + - 'export RUNNERTOKEN=$(curl -s -XPOST -H "authorization: token _REPLACE_TOKEN_" https://api.github.com/repos/coin-or/SHOT/actions/runners/registration-token | jq -r .token)' + - echo "Fetched token" + # Fetch the runner code + - su 'ghrunner' -c "curl -O -L https://github.com/actions/runner/releases/download/v2.302.1/actions-runner-linux-x64-2.302.1.tar.gz" + - echo "Fetched the installer" + # Extract the code as the runner user + - su 'ghrunner' -c "tar xzf ./actions-runner-linux-x64-2.302.1.tar.gz" + - echo "Installing the runner and dependencies" + # Config the runner and install it as a service. + - ./bin/installdependencies.sh + - su 'ghrunner' -c './config.sh --url https://github.com/coin-or/SHOT --token $RUNNERTOKEN --unattended --labels docker,x64,linux,cmake' + - ./svc.sh install ghrunner + - ./svc.sh start + - chown -R ghrunner /home/ghrunner/actions-runner + # Post install steps for docker + - groupadd docker + - usermod -aG docker ghrunner + - systemctl enable docker.service + - systemctl enable containerd.service From dfdfcac14007326c95999d7582d233118a48f657 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Fri, 17 Mar 2023 14:44:12 +0200 Subject: [PATCH 011/111] =?UTF-8?q?fix(ci):=20Fixed=20so=20the=20correct?= =?UTF-8?q?=20variables=20are=20used=20=E2=9C=8F=EF=B8=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index bd5e12d8..9d014664 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/download-artifact@v3 name: Download test results with: - name: ${needs.build-test.outputs.test_artifact} + name: ${{needs.build-test.outputs.test_artifact}} - name: Publish Test Results uses: EnricoMi/publish-unit-test-result-action@v2 From cf8d1e29e0a17686c129c0d4805844a80d3465f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Fri, 17 Mar 2023 14:46:45 +0200 Subject: [PATCH 012/111] fix: Fixed permission issues with docker and documented the runner-init.yml a bit better. --- misc/runner-init.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/misc/runner-init.yml b/misc/runner-init.yml index f23f1e56..29d0fb1c 100644 --- a/misc/runner-init.yml +++ b/misc/runner-init.yml @@ -3,15 +3,17 @@ package_update: true package_upgrade: true packages: - # Required for GitHub runner + # Requirements for GitHub runner - zlib1g - libssl1.1 - libkrb5-3 - liblttng-ust-ctl4 - liblttng-ust0 + - libssl-dev + # Required for the setup of GitHub Runner. - build-essential - jq - - libssl-dev + # Docker packages - ca-certificates - curl - gnupg @@ -66,3 +68,5 @@ runcmd: - usermod -aG docker ghrunner - systemctl enable docker.service - systemctl enable containerd.service + # Finally reboot the VM to correctly enable permissions for Docker. + - reboot now From 721ba040f77dc229a0571a48743dfa91f77578d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Fri, 17 Mar 2023 15:13:26 +0200 Subject: [PATCH 013/111] fix(ci): Attempt to use the correct output --- .github/workflows/build-push.yml | 2 +- .github/workflows/build-workflow.yml | 32 +++++++++++++--------------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 9d014664..db949e3f 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -12,8 +12,8 @@ jobs: retention: 30 publish-test: - needs: build-test name: Publish test results + needs: build-test runs-on: [self-hosted, docker] steps: - uses: actions/download-artifact@v3 diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index 4cd8582b..909041c6 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -25,21 +25,21 @@ on: required: false type: number outputs: - test_artifact: + test-artifact: description: "Test artifact ID" - value: ${{ jobs.build.outputs.test_artifact }} - options_artifact: + value: ${{ jobs.build.outputs.test-artifact }} + options-artifact: description: "SHOT Optionsfile artifact ID" - value: ${{ jobs.build.outputs.options_artifact }} - binary_artifact: + value: ${{ jobs.build.outputs.options-artifact }} + binary-artifact: description: "SHOT Binary artifact ID" - value: ${{ jobs.build.outputs.binary_artifact }} - package_artifact: + value: ${{ jobs.build.outputs.binary-artifact }} + package-artifact: description: "SHOT Package artifact ID" - value: ${{ jobs.build.outputs.package_artifact }} - library_artifact: + value: ${{ jobs.build.outputs.package-artifact }} + library-artifact: description: "SHOT Libraries artifact ID" - value: ${{ jobs.build.outputs.library_artifact }} + value: ${{ jobs.build.outputs.library-artifact }} env: @@ -51,11 +51,11 @@ env: jobs: build: outputs: - test_artifact: ${{ steps.artifact_outputs.test }} - options_artifact: ${{ steps.artifact_outputs.options }} - binary_artifact: ${{ steps.artifact_outputs.binary }} - package_artifact: ${{ steps.artifact_outputs.package }} - library_artifact: ${{ steps.artifact_outputs.library }} + test-artifact: ${{ steps.artifact_outputs.test }} + options-artifact: ${{ steps.artifact_outputs.options }} + binary-artifact: ${{ steps.artifact_outputs.binary }} + package-artifact: ${{ steps.artifact_outputs.package }} + library-artifact: ${{ steps.artifact_outputs.library }} name: Build SHOT runs-on: [self-hosted, linux] @@ -185,8 +185,6 @@ jobs: - name: Output artifact names shell: bash - # Execute tests defined by the CMake configuration. - # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail run: | echo "test=test-logs-${{ env.VERSION }}" >> $GITHUB_OUTPUT echo "options=shot-optsfile-${{ env.VERSION }}" >> $GITHUB_OUTPUT From cb1c0b42e5db0041dd3f99bfa999e22cd1fb280e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Mon, 20 Mar 2023 17:09:12 +0200 Subject: [PATCH 014/111] feat(ci): Allow for disabling artifact output and build args as inputs --- .github/workflows/build-push.yml | 33 ++++++++++++++-------------- .github/workflows/build-workflow.yml | 18 ++++++++++++++- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index db949e3f..a8ba7575 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -10,22 +10,23 @@ jobs: job_count: 3 build_type: Release retention: 30 + upload_artifacts: false - publish-test: - name: Publish test results - needs: build-test - runs-on: [self-hosted, docker] - steps: - - uses: actions/download-artifact@v3 - name: Download test results - with: - name: ${{needs.build-test.outputs.test_artifact}} - - - name: Publish Test Results - uses: EnricoMi/publish-unit-test-result-action@v2 - if: always() - with: - files: | - test-output/*.xml +# publish-test: +# name: Publish test results +# needs: build-test +# runs-on: [self-hosted, docker] +# steps: +# - uses: actions/download-artifact@v3 +# name: Download test results +# with: +# name: ${{needs.build-test.outputs.test_artifact}} +# +# - name: Publish Test Results +# uses: EnricoMi/publish-unit-test-result-action@v2 +# if: always() +# with: +# files: | +# test-output/*.xml diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index 909041c6..bb04753f 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -8,6 +8,11 @@ on: default: Release required: false type: string + build_args: + description: 'Additional build parameters' + default: '-DHAS_GAMS=off -DHAS_CPLEX=off -DHAS_GUROBI=off -DCOMPILE_TESTS=on -DSPDLOG_STATIC=on' + required: false + type: string job_count: description: 'The amount of jobs to use when building the program and dependencies.' default: 1 @@ -24,6 +29,11 @@ on: default: 60 required: false type: number + upload_artifacts: + description: 'Should the artifacts be uploaded' + default: true + required: false + type: boolean outputs: test-artifact: description: "Test artifact ID" @@ -47,6 +57,7 @@ env: BUILD_TYPE: ${{ inputs.build_type }} JOB_COUNT: ${{ inputs.job_count }} VERSION: ${{ inputs.version }} + EXTRA_BUILD_ARGS: ${{ inputs.build_args }} jobs: build: @@ -114,7 +125,7 @@ jobs: # Note the current convention is to use the -S and -B options here to specify source # and build directories, but this is only available with CMake 3.13 and higher. # The CMake binaries on the Github Actions machines are (as of this writing) 3.12 - run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DHAS_CBC=on -DCBC_DIR=${{runner.workspace}}/ThirdParty/Cbc -DHAS_IPOPT=ON -DIPOPT_DIR=${{runner.workspace}}/ThirdParty/Ipopt -DHAS_GAMS=off -DHAS_CPLEX=off -DHAS_GUROBI=off -DCOMPILE_TESTS=on -DSPDLOG_STATIC=on + run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DHAS_CBC=on -DCBC_DIR=${{runner.workspace}}/ThirdParty/Cbc -DHAS_IPOPT=ON -DIPOPT_DIR=${{runner.workspace}}/ThirdParty/Ipopt $EXTRA_BUILD_ARGS - name: Build working-directory: ${{runner.workspace}}/build @@ -154,6 +165,7 @@ jobs: retention-days: ${{ inputs.retention }} - uses: actions/upload-artifact@v3 + if: ${{ inputs.upload_artifacts }} name: Upload SHOT Optionsfile with: name: shot-optsfile-${{ env.VERSION }} @@ -161,6 +173,7 @@ jobs: retention-days: ${{ inputs.retention }} - uses: actions/upload-artifact@v3 + if: ${{ inputs.upload_artifacts }} name: Upload SHOT File with: name: shot-binary-${{ env.VERSION }} @@ -168,6 +181,7 @@ jobs: retention-days: ${{ inputs.retention }} - uses: actions/upload-artifact@v3 + if: ${{ inputs.upload_artifacts }} name: Upload SHOT Package file with: name: shot-package-${{ env.VERSION }} @@ -175,6 +189,7 @@ jobs: retention-days: ${{ inputs.retention }} - uses: actions/upload-artifact@v3 + if: ${{ inputs.upload_artifacts }} name: Upload SHOT Libraries with: name: shot-libraries-${{ env.VERSION }} @@ -184,6 +199,7 @@ jobs: retention-days: ${{ inputs.retention }} - name: Output artifact names + if: ${{ inputs.upload_artifacts }} shell: bash run: | echo "test=test-logs-${{ env.VERSION }}" >> $GITHUB_OUTPUT From fdc840a109c82e4169f154510241d4338e1f0dff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Tue, 21 Mar 2023 12:27:55 +0200 Subject: [PATCH 015/111] chore(ci): Formatting for the workflow --- .github/workflows/build-workflow.yml | 254 +++++++++++++-------------- 1 file changed, 127 insertions(+), 127 deletions(-) diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index bb04753f..e0b19f5e 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -68,135 +68,135 @@ jobs: package-artifact: ${{ steps.artifact_outputs.package }} library-artifact: ${{ steps.artifact_outputs.library }} name: Build SHOT - runs-on: [self-hosted, linux] + runs-on: [ self-hosted, linux ] steps: - - uses: actions/checkout@v3 - - - name: Checkout submodules - run: git submodule update --init --recursive - - - name: Create Build Environment - # Some projects don't allow in-source building, so create a separate build directory - # We'll use this as our working directory for all subsequent commands - run: cmake -E make_directory ${{runner.workspace}}/build - - - name: Install dependencies - shell: bash - run: sudo apt-get update ; sudo apt-get install -y libc6-dev libbz2-dev zlib1g-dev liblapack-dev libnauty2-dev libopenblas-base libopenblas-dev libmumps-dev pkgconf - - - name: Cache Cbc - id: cache-cbc - uses: actions/cache@v3 - with: - path: ${{runner.workspace}}/ThirdParty/Cbc - key: ${{ runner.os }}-cbc - - - name: Cache Ipopt - id: cache-ipopt - uses: actions/cache@v3 - with: - path: ${{runner.workspace}}/ThirdParty/Ipopt - key: ${{ runner.os }}-ipopt - - - name: Setup Coinbrew - if: steps.cache-ipopt.outputs.cache-hit != 'true' || steps.cache-cbc.outputs.cache-hit != 'true' - shell: bash - working-directory: ${{runner.workspace}}/build - run: git -C "coinbrew" pull || git clone https://github.com/coin-or/coinbrew - - - name: Download and build Ipopt - if: steps.cache-ipopt.outputs.cache-hit != 'true' - shell: bash - working-directory: ${{runner.workspace}}/build/coinbrew - run: ./coinbrew build Ipopt --verbosity 2 --prefix=${{runner.workspace}}/ThirdParty/Ipopt -j $JOB_COUNT --no-prompt --tests none - - - name: Download and build Cbc - if: steps.cache-cbc.outputs.cache-hit != 'true' - shell: bash - working-directory: ${{runner.workspace}}/build/coinbrew - run: rm -rf build dist ; ./coinbrew build Cbc@stable/2.10 --verbosity 2 -j $JOB_COUNT --prefix=${{runner.workspace}}/ThirdParty/Cbc --no-prompt --tests none --no-third-party - - - name: Configure CMake - # Use a bash shell so we can use the same syntax for environment variable - # access regardless of the host operating system - shell: bash - working-directory: ${{runner.workspace}}/build - # Note the current convention is to use the -S and -B options here to specify source - # and build directories, but this is only available with CMake 3.13 and higher. - # The CMake binaries on the Github Actions machines are (as of this writing) 3.12 - run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DHAS_CBC=on -DCBC_DIR=${{runner.workspace}}/ThirdParty/Cbc -DHAS_IPOPT=ON -DIPOPT_DIR=${{runner.workspace}}/ThirdParty/Ipopt $EXTRA_BUILD_ARGS - - - name: Build - working-directory: ${{runner.workspace}}/build - shell: bash - # Execute the build. You can specify a specific target with "--target " - run: cmake --build . --config $BUILD_TYPE -j $JOB_COUNT - - - name: Generate Optionsfile - working-directory: ${{runner.workspace}}/build - shell: bash - run: ./SHOT --opt - - - name: Test - id: test - working-directory: ${{runner.workspace}}/build - shell: bash - # Execute tests defined by the CMake configuration. - # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail - run: ctest -C $BUILD_TYPE --output-on-failure --output-junit test-output/ctest-junit-output.xml --output-log test-output/ctest-test.log - - - name: Construct package - working-directory: ${{runner.workspace}}/build - shell: bash - # Execute tests defined by the CMake configuration. - # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail - run: make package - - # This will always run as we always want the test output, however if the files are not found, we just warn the user. - - uses: actions/upload-artifact@v3 - if: ${{ always() }} - name: Upload test results - with: - name: test-logs-${{ env.VERSION }} - path: | - ${{runner.workspace}}/build/test-output/*.log - ${{runner.workspace}}/build/test-output/*.xml - retention-days: ${{ inputs.retention }} - - - uses: actions/upload-artifact@v3 - if: ${{ inputs.upload_artifacts }} - name: Upload SHOT Optionsfile - with: - name: shot-optsfile-${{ env.VERSION }} - path: ${{runner.workspace}}/build/SHOT.opt - retention-days: ${{ inputs.retention }} - - - uses: actions/upload-artifact@v3 - if: ${{ inputs.upload_artifacts }} - name: Upload SHOT File - with: - name: shot-binary-${{ env.VERSION }} - path: ${{runner.workspace}}/build/SHOT - retention-days: ${{ inputs.retention }} - - - uses: actions/upload-artifact@v3 - if: ${{ inputs.upload_artifacts }} - name: Upload SHOT Package file - with: - name: shot-package-${{ env.VERSION }} - path: ${{runner.workspace}}/build/SHOT-*-Linux.zip - retention-days: ${{ inputs.retention }} - - - uses: actions/upload-artifact@v3 - if: ${{ inputs.upload_artifacts }} - name: Upload SHOT Libraries - with: - name: shot-libraries-${{ env.VERSION }} - path: | - ${{runner.workspace}}/build/*.so - ${{runner.workspace}}/build/*.a - retention-days: ${{ inputs.retention }} + - uses: actions/checkout@v3 + + - name: Checkout submodules + run: git submodule update --init --recursive + + - name: Create Build Environment + # Some projects don't allow in-source building, so create a separate build directory + # We'll use this as our working directory for all subsequent commands + run: cmake -E make_directory ${{runner.workspace}}/build + + - name: Install dependencies + shell: bash + run: sudo apt-get update ; sudo apt-get install -y libc6-dev libbz2-dev zlib1g-dev liblapack-dev libnauty2-dev libopenblas-base libopenblas-dev libmumps-dev pkgconf + + - name: Cache Cbc + id: cache-cbc + uses: actions/cache@v3 + with: + path: ${{runner.workspace}}/ThirdParty/Cbc + key: ${{ runner.os }}-cbc + + - name: Cache Ipopt + id: cache-ipopt + uses: actions/cache@v3 + with: + path: ${{runner.workspace}}/ThirdParty/Ipopt + key: ${{ runner.os }}-ipopt + + - name: Setup Coinbrew + if: steps.cache-ipopt.outputs.cache-hit != 'true' || steps.cache-cbc.outputs.cache-hit != 'true' + shell: bash + working-directory: ${{runner.workspace}}/build + run: git -C "coinbrew" pull || git clone https://github.com/coin-or/coinbrew + + - name: Download and build Ipopt + if: steps.cache-ipopt.outputs.cache-hit != 'true' + shell: bash + working-directory: ${{runner.workspace}}/build/coinbrew + run: ./coinbrew build Ipopt --verbosity 2 --prefix=${{runner.workspace}}/ThirdParty/Ipopt -j $JOB_COUNT --no-prompt --tests none + + - name: Download and build Cbc + if: steps.cache-cbc.outputs.cache-hit != 'true' + shell: bash + working-directory: ${{runner.workspace}}/build/coinbrew + run: rm -rf build dist ; ./coinbrew build Cbc@stable/2.10 --verbosity 2 -j $JOB_COUNT --prefix=${{runner.workspace}}/ThirdParty/Cbc --no-prompt --tests none --no-third-party + + - name: Configure CMake + # Use a bash shell so we can use the same syntax for environment variable + # access regardless of the host operating system + shell: bash + working-directory: ${{runner.workspace}}/build + # Note the current convention is to use the -S and -B options here to specify source + # and build directories, but this is only available with CMake 3.13 and higher. + # The CMake binaries on the Github Actions machines are (as of this writing) 3.12 + run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DHAS_CBC=on -DCBC_DIR=${{runner.workspace}}/ThirdParty/Cbc -DHAS_IPOPT=ON -DIPOPT_DIR=${{runner.workspace}}/ThirdParty/Ipopt $EXTRA_BUILD_ARGS + + - name: Build + working-directory: ${{runner.workspace}}/build + shell: bash + # Execute the build. You can specify a specific target with "--target " + run: cmake --build . --config $BUILD_TYPE -j $JOB_COUNT + + - name: Generate Optionsfile + working-directory: ${{runner.workspace}}/build + shell: bash + run: ./SHOT --opt + + - name: Test + id: test + working-directory: ${{runner.workspace}}/build + shell: bash + # Execute tests defined by the CMake configuration. + # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail + run: ctest -C $BUILD_TYPE --output-on-failure --output-junit test-output/ctest-junit-output.xml --output-log test-output/ctest-test.log + + - name: Construct package + working-directory: ${{runner.workspace}}/build + shell: bash + # Execute tests defined by the CMake configuration. + # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail + run: make package + + # This will always run as we always want the test output, however if the files are not found, we just warn the user. + - uses: actions/upload-artifact@v3 + if: ${{ always() }} + name: Upload test results + with: + name: test-logs-${{ env.VERSION }} + path: | + ${{runner.workspace}}/build/test-output/*.log + ${{runner.workspace}}/build/test-output/*.xml + retention-days: ${{ inputs.retention }} + + - uses: actions/upload-artifact@v3 + if: ${{ inputs.upload_artifacts }} + name: Upload SHOT Optionsfile + with: + name: shot-optsfile-${{ env.VERSION }} + path: ${{runner.workspace}}/build/SHOT.opt + retention-days: ${{ inputs.retention }} + + - uses: actions/upload-artifact@v3 + if: ${{ inputs.upload_artifacts }} + name: Upload SHOT File + with: + name: shot-binary-${{ env.VERSION }} + path: ${{runner.workspace}}/build/SHOT + retention-days: ${{ inputs.retention }} + + - uses: actions/upload-artifact@v3 + if: ${{ inputs.upload_artifacts }} + name: Upload SHOT Package file + with: + name: shot-package-${{ env.VERSION }} + path: ${{runner.workspace}}/build/SHOT-*-Linux.zip + retention-days: ${{ inputs.retention }} + + - uses: actions/upload-artifact@v3 + if: ${{ inputs.upload_artifacts }} + name: Upload SHOT Libraries + with: + name: shot-libraries-${{ env.VERSION }} + path: | + ${{runner.workspace}}/build/*.so + ${{runner.workspace}}/build/*.a + retention-days: ${{ inputs.retention }} - name: Output artifact names if: ${{ inputs.upload_artifacts }} From 884d000f518edb7dba75a2f679b3d1f22a42ed61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Tue, 21 Mar 2023 12:28:29 +0200 Subject: [PATCH 016/111] feat(ci): Output test artifact name even if the other artifacts are not to be uploaded --- .github/workflows/build-workflow.yml | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index e0b19f5e..240a5d22 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -198,12 +198,18 @@ jobs: ${{runner.workspace}}/build/*.a retention-days: ${{ inputs.retention }} - - name: Output artifact names - if: ${{ inputs.upload_artifacts }} - shell: bash - run: | - echo "test=test-logs-${{ env.VERSION }}" >> $GITHUB_OUTPUT - echo "options=shot-optsfile-${{ env.VERSION }}" >> $GITHUB_OUTPUT - echo "binary=shot-binary-${{ env.VERSION }}" >> $GITHUB_OUTPUT - echo "package=shot-package-${{ env.VERSION }}" >> $GITHUB_OUTPUT - echo "library=shot-libraries-${{ env.VERSION }}" >> $GITHUB_OUTPUT + - name: Output artifact names + id: output-artifacts + if: ${{ inputs.upload_artifacts }} + shell: bash + run: | + echo "test=test-logs-${{ env.VERSION }}" >> $GITHUB_OUTPUT + echo "options=shot-optsfile-${{ env.VERSION }}" >> $GITHUB_OUTPUT + echo "binary=shot-binary-${{ env.VERSION }}" >> $GITHUB_OUTPUT + echo "package=shot-package-${{ env.VERSION }}" >> $GITHUB_OUTPUT + echo "library=shot-libraries-${{ env.VERSION }}" >> $GITHUB_OUTPUT + + - name: Output test artifacts + if: ${{ inputs.upload_artifacts }} != true && steps.output-artifacts.outcome != 'success' + shell: bash + run: echo "test=test-logs-${{ env.VERSION }}" >> $GITHUB_OUTPUT From 50070ed0cc0c70417917fb2ea29f6bfd8e46cb82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Tue, 21 Mar 2023 14:03:32 +0200 Subject: [PATCH 017/111] refactor(ci): refactored runner.workspace -> github.workspace --- .github/workflows/build-workflow.yml | 40 ++++++++++++++-------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index 240a5d22..0cf688c8 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -68,7 +68,7 @@ jobs: package-artifact: ${{ steps.artifact_outputs.package }} library-artifact: ${{ steps.artifact_outputs.library }} name: Build SHOT - runs-on: [ self-hosted, linux ] + runs-on: [ self-hosted, linux, cmake ] steps: - uses: actions/checkout@v3 @@ -79,7 +79,7 @@ jobs: - name: Create Build Environment # Some projects don't allow in-source building, so create a separate build directory # We'll use this as our working directory for all subsequent commands - run: cmake -E make_directory ${{runner.workspace}}/build + run: cmake -E make_directory ${{github.workspace}}/build - name: Install dependencies shell: bash @@ -89,27 +89,27 @@ jobs: id: cache-cbc uses: actions/cache@v3 with: - path: ${{runner.workspace}}/ThirdParty/Cbc + path: ${{github.workspace}}/ThirdParty/Cbc key: ${{ runner.os }}-cbc - name: Cache Ipopt id: cache-ipopt uses: actions/cache@v3 with: - path: ${{runner.workspace}}/ThirdParty/Ipopt + path: ${{github.workspace}}/ThirdParty/Ipopt key: ${{ runner.os }}-ipopt - name: Setup Coinbrew if: steps.cache-ipopt.outputs.cache-hit != 'true' || steps.cache-cbc.outputs.cache-hit != 'true' shell: bash - working-directory: ${{runner.workspace}}/build + working-directory: ${{github.workspace}}/build run: git -C "coinbrew" pull || git clone https://github.com/coin-or/coinbrew - name: Download and build Ipopt if: steps.cache-ipopt.outputs.cache-hit != 'true' shell: bash - working-directory: ${{runner.workspace}}/build/coinbrew - run: ./coinbrew build Ipopt --verbosity 2 --prefix=${{runner.workspace}}/ThirdParty/Ipopt -j $JOB_COUNT --no-prompt --tests none + working-directory: ${{github.workspace}}/build/coinbrew + run: ./coinbrew build Ipopt --verbosity 2 --prefix=${{github.workspace}}/ThirdParty/Ipopt -j $JOB_COUNT --no-prompt --tests none - name: Download and build Cbc if: steps.cache-cbc.outputs.cache-hit != 'true' @@ -121,33 +121,33 @@ jobs: # Use a bash shell so we can use the same syntax for environment variable # access regardless of the host operating system shell: bash - working-directory: ${{runner.workspace}}/build + working-directory: ${{github.workspace}}/build # Note the current convention is to use the -S and -B options here to specify source # and build directories, but this is only available with CMake 3.13 and higher. # The CMake binaries on the Github Actions machines are (as of this writing) 3.12 - run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DHAS_CBC=on -DCBC_DIR=${{runner.workspace}}/ThirdParty/Cbc -DHAS_IPOPT=ON -DIPOPT_DIR=${{runner.workspace}}/ThirdParty/Ipopt $EXTRA_BUILD_ARGS + run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DHAS_CBC=on -DCBC_DIR=${{github.workspace}}/ThirdParty/Cbc -DHAS_IPOPT=ON -DIPOPT_DIR=${{github.workspace}}/ThirdParty/Ipopt $EXTRA_BUILD_ARGS - name: Build - working-directory: ${{runner.workspace}}/build + working-directory: ${{github.workspace}}/build shell: bash # Execute the build. You can specify a specific target with "--target " run: cmake --build . --config $BUILD_TYPE -j $JOB_COUNT - name: Generate Optionsfile - working-directory: ${{runner.workspace}}/build + working-directory: ${{github.workspace}}/build shell: bash run: ./SHOT --opt - name: Test id: test - working-directory: ${{runner.workspace}}/build + working-directory: ${{github.workspace}}/build shell: bash # Execute tests defined by the CMake configuration. # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail run: ctest -C $BUILD_TYPE --output-on-failure --output-junit test-output/ctest-junit-output.xml --output-log test-output/ctest-test.log - name: Construct package - working-directory: ${{runner.workspace}}/build + working-directory: ${{github.workspace}}/build shell: bash # Execute tests defined by the CMake configuration. # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail @@ -160,8 +160,8 @@ jobs: with: name: test-logs-${{ env.VERSION }} path: | - ${{runner.workspace}}/build/test-output/*.log - ${{runner.workspace}}/build/test-output/*.xml + ${{github.workspace}}/build/test-output/*.log + ${{github.workspace}}/build/test-output/*.xml retention-days: ${{ inputs.retention }} - uses: actions/upload-artifact@v3 @@ -169,7 +169,7 @@ jobs: name: Upload SHOT Optionsfile with: name: shot-optsfile-${{ env.VERSION }} - path: ${{runner.workspace}}/build/SHOT.opt + path: ${{github.workspace}}/build/SHOT.opt retention-days: ${{ inputs.retention }} - uses: actions/upload-artifact@v3 @@ -177,7 +177,7 @@ jobs: name: Upload SHOT File with: name: shot-binary-${{ env.VERSION }} - path: ${{runner.workspace}}/build/SHOT + path: ${{github.workspace}}/build/SHOT retention-days: ${{ inputs.retention }} - uses: actions/upload-artifact@v3 @@ -185,7 +185,7 @@ jobs: name: Upload SHOT Package file with: name: shot-package-${{ env.VERSION }} - path: ${{runner.workspace}}/build/SHOT-*-Linux.zip + path: ${{github.workspace}}/build/SHOT-*-Linux.zip retention-days: ${{ inputs.retention }} - uses: actions/upload-artifact@v3 @@ -194,8 +194,8 @@ jobs: with: name: shot-libraries-${{ env.VERSION }} path: | - ${{runner.workspace}}/build/*.so - ${{runner.workspace}}/build/*.a + ${{github.workspace}}/build/*.so + ${{github.workspace}}/build/*.a retention-days: ${{ inputs.retention }} - name: Output artifact names From 03af1125d629efdfa440af26dbcaa9806b136b16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Tue, 21 Mar 2023 14:05:25 +0200 Subject: [PATCH 018/111] feat(ci): Added ability to specify proprietary build and added gurobi download and license file setup --- .github/workflows/build-push.yml | 47 +++++++++++++++++----------- .github/workflows/build-workflow.yml | 45 ++++++++++++++++++++++++-- 2 files changed, 72 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index a8ba7575..da7931f7 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -1,6 +1,6 @@ name: Build -on: [push] +on: [ push ] jobs: build-test: @@ -10,23 +10,34 @@ jobs: job_count: 3 build_type: Release retention: 30 - upload_artifacts: false + upload_artifacts: true + + publish-test: + name: Publish test results + needs: build-test + runs-on: [ self-hosted, docker ] + steps: + - uses: actions/download-artifact@v3 + name: Download test results + with: + name: ${{needs.build-test.outputs.test_artifact}} -# publish-test: -# name: Publish test results -# needs: build-test -# runs-on: [self-hosted, docker] -# steps: -# - uses: actions/download-artifact@v3 -# name: Download test results -# with: -# name: ${{needs.build-test.outputs.test_artifact}} -# -# - name: Publish Test Results -# uses: EnricoMi/publish-unit-test-result-action@v2 -# if: always() -# with: -# files: | -# test-output/*.xml + - name: Publish Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + files: | + build/test-output/*.xml + + build-properitary: + name: Build & Test (Proprietary licenses) + needs: [ build-test ] + uses: ./.github/workflows/build-workflow.yml + with: + job_count: 3 + build_type: Release + retention: 30 + upload_artifacts: false + is_proprietary: true diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index 0cf688c8..c3593cc4 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -34,6 +34,12 @@ on: default: true required: false type: boolean + is_proprietary: + description: 'Is this a proprietary build (this will install Gurobi and GAMS)' + default: false + required: false + type: boolean + outputs: test-artifact: description: "Test artifact ID" @@ -58,6 +64,8 @@ env: JOB_COUNT: ${{ inputs.job_count }} VERSION: ${{ inputs.version }} EXTRA_BUILD_ARGS: ${{ inputs.build_args }} + GRB_LICENSE_FILE: ${{ github.workspace }}/gurobi.lic + GRB_INSTALL_PATH: ${{ github.workspace }}/ThirdParty/gurobi jobs: build: @@ -71,6 +79,13 @@ jobs: runs-on: [ self-hosted, linux, cmake ] steps: + - name: Cleanup workspace + run: | + ls -la ./ + rm -rf ./* || true + rm -rf ./.??* || true + ls -la ./ + - uses: actions/checkout@v3 - name: Checkout submodules @@ -114,8 +129,29 @@ jobs: - name: Download and build Cbc if: steps.cache-cbc.outputs.cache-hit != 'true' shell: bash - working-directory: ${{runner.workspace}}/build/coinbrew - run: rm -rf build dist ; ./coinbrew build Cbc@stable/2.10 --verbosity 2 -j $JOB_COUNT --prefix=${{runner.workspace}}/ThirdParty/Cbc --no-prompt --tests none --no-third-party + working-directory: ${{github.workspace}}/build/coinbrew + run: rm -rf build dist ; ./coinbrew build Cbc@stable/2.10 --verbosity 2 -j $JOB_COUNT --prefix=${{github.workspace}}/ThirdParty/Cbc --no-prompt --tests none --no-third-party + + - name: Setup Gurobi License file + if: ${{ inputs.is_proprietary }} + shell: bash + working-directory: ${{github.workspace}} + run: echo ${{ secrets.GUROBI_LICENSE_FILE }} | base64 -d > gurobi.lic + + - name: Download Gurobi + if: ${{ inputs.is_proprietary }} + shell: bash + working-directory: ${{github.workspace}}/ThirdParty + run: wget https://packages.gurobi.com/10.0/gurobi10.0.1_linux64.tar.gz ; tar -xvf gurobi10.0.1_linux64.tar.gz ; rm gurobi10.0.1_linux64.tar.gz + + - name: Setup Gurobi environment variables + if: ${{ inputs.is_proprietary }} + shell: bash + working-directory: ${{github.workspace}}/ThirdParty + run: | + echo "${{ github.workspace }}/ThirdParty/gurobi1001/linux64/bin" >> $GITHUB_PATH + echo "GUROBI_HOME=${{ github.workspace }}/ThirdParty/gurobi1001/linux64" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=${{ github.workspace }}/ThirdParty/gurobi1001/linux64/lib" >> $GITHUB_ENV - name: Configure CMake # Use a bash shell so we can use the same syntax for environment variable @@ -213,3 +249,8 @@ jobs: if: ${{ inputs.upload_artifacts }} != true && steps.output-artifacts.outcome != 'success' shell: bash run: echo "test=test-logs-${{ env.VERSION }}" >> $GITHUB_OUTPUT + + - name: Cleanup license files + if: ${{ inputs.is_proprietary }} + shell: bash + run: rm -f $GRB_LICENSE_FILE From 7a175fbc4efdb73fd9d08ecb4e1c42e79dc5765d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Tue, 21 Mar 2023 14:34:36 +0200 Subject: [PATCH 019/111] feat(ci): Updated the reusable workflow to correctly get secrets --- .github/workflows/build-push.yml | 3 +++ .github/workflows/build-workflow.yml | 9 ++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index da7931f7..bc8dbd12 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -40,4 +40,7 @@ jobs: retention: 30 upload_artifacts: false is_proprietary: true + secrets: + gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} + gams_license: ${{ secrets.GAMS_LICENSE_FILE }} diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index c3593cc4..829d4dbc 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -39,6 +39,13 @@ on: default: false required: false type: boolean + secrets: + gurobi_license: + description: 'Gurobi license file secret' + required: false + gams_license: + description: 'GAMS license file secret' + required: false outputs: test-artifact: @@ -136,7 +143,7 @@ jobs: if: ${{ inputs.is_proprietary }} shell: bash working-directory: ${{github.workspace}} - run: echo ${{ secrets.GUROBI_LICENSE_FILE }} | base64 -d > gurobi.lic + run: echo ${{ secrets.gurobi_license }} | base64 -d > gurobi.lic - name: Download Gurobi if: ${{ inputs.is_proprietary }} From a2abadc190766769e0a388b24339205e63cad8a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Tue, 21 Mar 2023 14:42:03 +0200 Subject: [PATCH 020/111] feat(ci): Actually compile with gurobi as a test --- .github/workflows/build-push.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index bc8dbd12..e870b1cd 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -40,6 +40,7 @@ jobs: retention: 30 upload_artifacts: false is_proprietary: true + build_args: '-DHAS_GAMS=off -DHAS_CPLEX=off -DHAS_GUROBI=on -DGUROBI_DIR=$GRB_INSTALL_PATH -DCOMPILE_TESTS=on -DSPDLOG_STATIC=on' secrets: gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} gams_license: ${{ secrets.GAMS_LICENSE_FILE }} From c676c1100424a7a5f59c616fba5229032c0796d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Tue, 21 Mar 2023 15:09:15 +0200 Subject: [PATCH 021/111] chore(ci): Updated the job count to match the current runner --- .github/workflows/build-push.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index e870b1cd..ed3b51d8 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -7,7 +7,7 @@ jobs: name: Build & Test uses: ./.github/workflows/build-workflow.yml with: - job_count: 3 + job_count: 4 build_type: Release retention: 30 upload_artifacts: true @@ -35,7 +35,7 @@ jobs: needs: [ build-test ] uses: ./.github/workflows/build-workflow.yml with: - job_count: 3 + job_count: 4 build_type: Release retention: 30 upload_artifacts: false From 0fff51f1a72d021a2b47c111e09d28cce535d9a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Tue, 21 Mar 2023 15:12:46 +0200 Subject: [PATCH 022/111] chore(ci): Removed debug from cleanup --- .github/workflows/build-workflow.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index 829d4dbc..90f29ad2 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -88,10 +88,8 @@ jobs: steps: - name: Cleanup workspace run: | - ls -la ./ rm -rf ./* || true rm -rf ./.??* || true - ls -la ./ - uses: actions/checkout@v3 From 1a5ea8808fe4e671473aedddfe59de32f584959f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Tue, 21 Mar 2023 15:58:30 +0200 Subject: [PATCH 023/111] feat(ci): Added GAMS installation steps and compile using it --- .github/workflows/build-push.yml | 2 +- .github/workflows/build-workflow.yml | 34 ++++++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index ed3b51d8..7715ccd8 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -40,7 +40,7 @@ jobs: retention: 30 upload_artifacts: false is_proprietary: true - build_args: '-DHAS_GAMS=off -DHAS_CPLEX=off -DHAS_GUROBI=on -DGUROBI_DIR=$GRB_INSTALL_PATH -DCOMPILE_TESTS=on -DSPDLOG_STATIC=on' + build_args: '-DHAS_GAMS=on -DGAMS_DIR=$GAMS_INSTALL_PATH -DHAS_CPLEX=off -DHAS_GUROBI=on -DGUROBI_DIR=$GRB_INSTALL_PATH -DCOMPILE_TESTS=on -DSPDLOG_STATIC=on' secrets: gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} gams_license: ${{ secrets.GAMS_LICENSE_FILE }} diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index 90f29ad2..e0f55bfc 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -73,6 +73,8 @@ env: EXTRA_BUILD_ARGS: ${{ inputs.build_args }} GRB_LICENSE_FILE: ${{ github.workspace }}/gurobi.lic GRB_INSTALL_PATH: ${{ github.workspace }}/ThirdParty/gurobi + GAMS_LICENSE_FILE: ${{ github.workspace }}/ThirdParty/gams42.3/gamslice.txt + GAMS_INSTALL_PATH: ${{ github.workspace }}/ThirdParty/gams42.3/ jobs: build: @@ -137,18 +139,22 @@ jobs: working-directory: ${{github.workspace}}/build/coinbrew run: rm -rf build dist ; ./coinbrew build Cbc@stable/2.10 --verbosity 2 -j $JOB_COUNT --prefix=${{github.workspace}}/ThirdParty/Cbc --no-prompt --tests none --no-third-party + # Proprietary software setup happens here (GAMS and Gurobi) + # First we create the license files from the base64 encoded secrets. - name: Setup Gurobi License file if: ${{ inputs.is_proprietary }} shell: bash working-directory: ${{github.workspace}} - run: echo ${{ secrets.gurobi_license }} | base64 -d > gurobi.lic + run: echo ${{ secrets.gurobi_license }} | base64 -d > $GRB_LICENSE_FILE + # Then we download and install Gurobi - name: Download Gurobi if: ${{ inputs.is_proprietary }} shell: bash working-directory: ${{github.workspace}}/ThirdParty run: wget https://packages.gurobi.com/10.0/gurobi10.0.1_linux64.tar.gz ; tar -xvf gurobi10.0.1_linux64.tar.gz ; rm gurobi10.0.1_linux64.tar.gz + # Finally, we set the environment variables needed for Gurobi to work. - name: Setup Gurobi environment variables if: ${{ inputs.is_proprietary }} shell: bash @@ -158,6 +164,28 @@ jobs: echo "GUROBI_HOME=${{ github.workspace }}/ThirdParty/gurobi1001/linux64" >> $GITHUB_ENV echo "LD_LIBRARY_PATH=${{ github.workspace }}/ThirdParty/gurobi1001/linux64/lib" >> $GITHUB_ENV + # Next, we set up GAMS + + # Download GAMS + - name: Download GAMS + if: ${{ inputs.is_proprietary }} + shell: bash + working-directory: ${{github.workspace}}/ThirdParty + run: wget https://d37drm4t2jghv5.cloudfront.net/distributions/42.3.0/linux/linux_x64_64_sfx.exe ; chmod +x linux_x64_64_sfx.exe ; ./linux_x64_64_sfx.exe -d gams42.3 ; rm linux_x64_64_sfx.exe + + # Add the license file to the GAMS installation directory + - name: Setup GAMS License file + if: ${{ inputs.is_proprietary }} + shell: bash + working-directory: ${{github.workspace}} + run: echo ${{ secrets.gams_license }} | base64 -d > $GAMS_LICENSE_FILE + + - name: Install GAMS + if: ${{ inputs.is_proprietary }} + shell: bash + working-directory: ${{ github.workspace }}/ThirdParty/gams42.3/ + run: ./gamsinst -a + - name: Configure CMake # Use a bash shell so we can use the same syntax for environment variable # access regardless of the host operating system @@ -258,4 +286,6 @@ jobs: - name: Cleanup license files if: ${{ inputs.is_proprietary }} shell: bash - run: rm -f $GRB_LICENSE_FILE + run: | + rm -f $GRB_LICENSE_FILE + rm -f $GAMS_LICENSE_FILE From 1e4ae738f945dc5c2dd58c668404b5a404d2a56e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Tue, 21 Mar 2023 16:03:40 +0200 Subject: [PATCH 024/111] feat(ci): Added concurrency, some comments and ability to add suffix to artifacts. --- .github/workflows/build-push.yml | 9 ++++++++- .github/workflows/build-workflow.yml | 16 +++++++++++----- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 7715ccd8..cce60850 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -1,8 +1,13 @@ name: Build +# Build this on push on: [ push ] +# Only build one build at a time +concurrency: push-build + jobs: + # First, we build and test using the default build options and only open source external libraries build-test: name: Build & Test uses: ./.github/workflows/build-workflow.yml @@ -12,6 +17,7 @@ jobs: retention: 30 upload_artifacts: true + # Publish the test results as an output publish-test: name: Publish test results needs: build-test @@ -28,8 +34,8 @@ jobs: with: files: | build/test-output/*.xml - + # Then, we build and test using all licenses included proprietary ones build-properitary: name: Build & Test (Proprietary licenses) needs: [ build-test ] @@ -40,6 +46,7 @@ jobs: retention: 30 upload_artifacts: false is_proprietary: true + artifact_suffix: '-proprietary' build_args: '-DHAS_GAMS=on -DGAMS_DIR=$GAMS_INSTALL_PATH -DHAS_CPLEX=off -DHAS_GUROBI=on -DGUROBI_DIR=$GRB_INSTALL_PATH -DCOMPILE_TESTS=on -DSPDLOG_STATIC=on' secrets: gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index e0f55bfc..0d8237f8 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -39,6 +39,11 @@ on: default: false required: false type: boolean + artifact_suffix: + description: 'The suffix to add to the artifact name' + default: '' + required: false + type: string secrets: gurobi_license: description: 'Gurobi license file secret' @@ -70,6 +75,7 @@ env: BUILD_TYPE: ${{ inputs.build_type }} JOB_COUNT: ${{ inputs.job_count }} VERSION: ${{ inputs.version }} + ARTIFACT_SUFFIX: ${{ inputs.artifact_suffix }} EXTRA_BUILD_ARGS: ${{ inputs.build_args }} GRB_LICENSE_FILE: ${{ github.workspace }}/gurobi.lic GRB_INSTALL_PATH: ${{ github.workspace }}/ThirdParty/gurobi @@ -227,7 +233,7 @@ jobs: if: ${{ always() }} name: Upload test results with: - name: test-logs-${{ env.VERSION }} + name: test-logs-${{ env.VERSION }}${{ env.ARTIFACT_SUFFIX }} path: | ${{github.workspace}}/build/test-output/*.log ${{github.workspace}}/build/test-output/*.xml @@ -237,7 +243,7 @@ jobs: if: ${{ inputs.upload_artifacts }} name: Upload SHOT Optionsfile with: - name: shot-optsfile-${{ env.VERSION }} + name: shot-optsfile-${{ env.VERSION }}${{ env.ARTIFACT_SUFFIX }} path: ${{github.workspace}}/build/SHOT.opt retention-days: ${{ inputs.retention }} @@ -245,7 +251,7 @@ jobs: if: ${{ inputs.upload_artifacts }} name: Upload SHOT File with: - name: shot-binary-${{ env.VERSION }} + name: shot-binary-${{ env.VERSION }}${{ env.ARTIFACT_SUFFIX }} path: ${{github.workspace}}/build/SHOT retention-days: ${{ inputs.retention }} @@ -253,7 +259,7 @@ jobs: if: ${{ inputs.upload_artifacts }} name: Upload SHOT Package file with: - name: shot-package-${{ env.VERSION }} + name: shot-package-${{ env.VERSION }}${{ env.ARTIFACT_SUFFIX }} path: ${{github.workspace}}/build/SHOT-*-Linux.zip retention-days: ${{ inputs.retention }} @@ -261,7 +267,7 @@ jobs: if: ${{ inputs.upload_artifacts }} name: Upload SHOT Libraries with: - name: shot-libraries-${{ env.VERSION }} + name: shot-libraries-${{ env.VERSION }}${{ env.ARTIFACT_SUFFIX }} path: | ${{github.workspace}}/build/*.so ${{github.workspace}}/build/*.a From ff5b695aaadb3ed561c78ef438ae890b6227d6b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Tue, 21 Mar 2023 16:07:37 +0200 Subject: [PATCH 025/111] fix(ci): Fixed the publish test job to use correct outputs --- .github/workflows/build-push.yml | 2 +- .github/workflows/build-workflow.yml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index cce60850..2fbcc65a 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -26,7 +26,7 @@ jobs: - uses: actions/download-artifact@v3 name: Download test results with: - name: ${{needs.build-test.outputs.test_artifact}} + name: ${{ needs.build-test.outputs.test-artifact }} - name: Publish Test Results uses: EnricoMi/publish-unit-test-result-action@v2 diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index 0d8237f8..9bb9e979 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -71,7 +71,6 @@ on: env: - # Customize BUILD_TYPE: ${{ inputs.build_type }} JOB_COUNT: ${{ inputs.job_count }} VERSION: ${{ inputs.version }} From b1a7cfea9e63da0ac18b60864d28ff767cb4d062 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Tue, 21 Mar 2023 16:13:54 +0200 Subject: [PATCH 026/111] fix(ci): Fixed GAMS installation as it always extracts using the same name --- .github/workflows/build-workflow.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index 9bb9e979..470571a1 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -78,8 +78,8 @@ env: EXTRA_BUILD_ARGS: ${{ inputs.build_args }} GRB_LICENSE_FILE: ${{ github.workspace }}/gurobi.lic GRB_INSTALL_PATH: ${{ github.workspace }}/ThirdParty/gurobi - GAMS_LICENSE_FILE: ${{ github.workspace }}/ThirdParty/gams42.3/gamslice.txt - GAMS_INSTALL_PATH: ${{ github.workspace }}/ThirdParty/gams42.3/ + GAMS_LICENSE_FILE: ${{ github.workspace }}/ThirdParty/gams42.3_linux_x64_64_sfx/gamslice.txt + GAMS_INSTALL_PATH: ${{ github.workspace }}/ThirdParty/gams42.3_linux_x64_64_sfx/ jobs: build: @@ -176,7 +176,7 @@ jobs: if: ${{ inputs.is_proprietary }} shell: bash working-directory: ${{github.workspace}}/ThirdParty - run: wget https://d37drm4t2jghv5.cloudfront.net/distributions/42.3.0/linux/linux_x64_64_sfx.exe ; chmod +x linux_x64_64_sfx.exe ; ./linux_x64_64_sfx.exe -d gams42.3 ; rm linux_x64_64_sfx.exe + run: wget https://d37drm4t2jghv5.cloudfront.net/distributions/42.3.0/linux/linux_x64_64_sfx.exe ; chmod +x linux_x64_64_sfx.exe ; ./linux_x64_64_sfx.exe ; rm linux_x64_64_sfx.exe # Add the license file to the GAMS installation directory - name: Setup GAMS License file @@ -188,7 +188,7 @@ jobs: - name: Install GAMS if: ${{ inputs.is_proprietary }} shell: bash - working-directory: ${{ github.workspace }}/ThirdParty/gams42.3/ + working-directory: ${{ github.workspace }}/ThirdParty/gams42.3_linux_x64_64_sfx/ run: ./gamsinst -a - name: Configure CMake @@ -289,7 +289,7 @@ jobs: run: echo "test=test-logs-${{ env.VERSION }}" >> $GITHUB_OUTPUT - name: Cleanup license files - if: ${{ inputs.is_proprietary }} + if: ${{ success() || failure() }} shell: bash run: | rm -f $GRB_LICENSE_FILE From 5192d101fa2534ae25b0acd8461c2a797918877f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Tue, 21 Mar 2023 16:37:40 +0200 Subject: [PATCH 027/111] chore(ci): Formatting the workflow --- .github/workflows/build-workflow.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index 470571a1..efb52764 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -170,7 +170,6 @@ jobs: echo "LD_LIBRARY_PATH=${{ github.workspace }}/ThirdParty/gurobi1001/linux64/lib" >> $GITHUB_ENV # Next, we set up GAMS - # Download GAMS - name: Download GAMS if: ${{ inputs.is_proprietary }} From e55148ef1704b7fe5dcdc0e0c7a139d6b757fb26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Tue, 21 Mar 2023 16:38:45 +0200 Subject: [PATCH 028/111] fix(ci): Use the correct format for the GAMS library --- .github/workflows/build-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index efb52764..ead87e9f 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -79,7 +79,7 @@ env: GRB_LICENSE_FILE: ${{ github.workspace }}/gurobi.lic GRB_INSTALL_PATH: ${{ github.workspace }}/ThirdParty/gurobi GAMS_LICENSE_FILE: ${{ github.workspace }}/ThirdParty/gams42.3_linux_x64_64_sfx/gamslice.txt - GAMS_INSTALL_PATH: ${{ github.workspace }}/ThirdParty/gams42.3_linux_x64_64_sfx/ + GAMS_INSTALL_PATH: ${{ github.workspace }}/ThirdParty/gams42.3_linux_x64_64_sfx jobs: build: From 0837d1aed994d37b5816cb18bd0deb625c6fa0d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Tue, 21 Mar 2023 16:44:20 +0200 Subject: [PATCH 029/111] feat(ci): Added GAMS path to the runner path --- .github/workflows/build-workflow.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index ead87e9f..df6bfe37 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -190,6 +190,11 @@ jobs: working-directory: ${{ github.workspace }}/ThirdParty/gams42.3_linux_x64_64_sfx/ run: ./gamsinst -a + - name: Add GAMS to the path + if: ${{ inputs.is_proprietary }} + shell: bash + run: echo "$GAMS_INSTALL_PATH" >> $GITHUB_PATH + - name: Configure CMake # Use a bash shell so we can use the same syntax for environment variable # access regardless of the host operating system From fa29e0d9709c2587c061a1fec8eb62de70311d49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Wed, 22 Mar 2023 15:52:38 +0200 Subject: [PATCH 030/111] fix(ci): Correctly use the gurobi path --- .github/workflows/build-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index df6bfe37..44ae64ce 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -77,7 +77,7 @@ env: ARTIFACT_SUFFIX: ${{ inputs.artifact_suffix }} EXTRA_BUILD_ARGS: ${{ inputs.build_args }} GRB_LICENSE_FILE: ${{ github.workspace }}/gurobi.lic - GRB_INSTALL_PATH: ${{ github.workspace }}/ThirdParty/gurobi + GRB_INSTALL_PATH: ${{ github.workspace }}/ThirdParty/gurobi1001 GAMS_LICENSE_FILE: ${{ github.workspace }}/ThirdParty/gams42.3_linux_x64_64_sfx/gamslice.txt GAMS_INSTALL_PATH: ${{ github.workspace }}/ThirdParty/gams42.3_linux_x64_64_sfx From 0e7270c36755742f6a713b145c3fc24799140f51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Wed, 22 Mar 2023 16:07:22 +0200 Subject: [PATCH 031/111] fix(ci): Attempt to escape the environment variables so they are correctly read by bash --- .github/workflows/build-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 2fbcc65a..38e20602 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -47,7 +47,7 @@ jobs: upload_artifacts: false is_proprietary: true artifact_suffix: '-proprietary' - build_args: '-DHAS_GAMS=on -DGAMS_DIR=$GAMS_INSTALL_PATH -DHAS_CPLEX=off -DHAS_GUROBI=on -DGUROBI_DIR=$GRB_INSTALL_PATH -DCOMPILE_TESTS=on -DSPDLOG_STATIC=on' + build_args: '-DHAS_GAMS=on -DGAMS_DIR=${GAMS_INSTALL_PATH} -DHAS_CPLEX=off -DHAS_GUROBI=on -DGUROBI_DIR=${GRB_INSTALL_PATH} -DCOMPILE_TESTS=on -DSPDLOG_STATIC=on' secrets: gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} gams_license: ${{ secrets.GAMS_LICENSE_FILE }} From 9ec908c322a598da7620265b62d976e42cff9b26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Wed, 22 Mar 2023 16:25:54 +0200 Subject: [PATCH 032/111] refactor(ci): Always pass the installation paths of the proprietary software directly in the workflow --- .github/workflows/build-push.yml | 58 ++++++++++++++-------------- .github/workflows/build-workflow.yml | 2 +- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 38e20602..e906cab1 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -7,38 +7,38 @@ on: [ push ] concurrency: push-build jobs: - # First, we build and test using the default build options and only open source external libraries - build-test: - name: Build & Test - uses: ./.github/workflows/build-workflow.yml - with: - job_count: 4 - build_type: Release - retention: 30 - upload_artifacts: true - - # Publish the test results as an output - publish-test: - name: Publish test results - needs: build-test - runs-on: [ self-hosted, docker ] - steps: - - uses: actions/download-artifact@v3 - name: Download test results - with: - name: ${{ needs.build-test.outputs.test-artifact }} - - - name: Publish Test Results - uses: EnricoMi/publish-unit-test-result-action@v2 - if: always() - with: - files: | - build/test-output/*.xml +# # First, we build and test using the default build options and only open source external libraries +# build-test: +# name: Build & Test +# uses: ./.github/workflows/build-workflow.yml +# with: +# job_count: 4 +# build_type: Release +# retention: 30 +# upload_artifacts: true +# +# # Publish the test results as an output +# publish-test: +# name: Publish test results +# needs: build-test +# runs-on: [ self-hosted, docker ] +# steps: +# - uses: actions/download-artifact@v3 +# name: Download test results +# with: +# name: ${{ needs.build-test.outputs.test-artifact }} +# +# - name: Publish Test Results +# uses: EnricoMi/publish-unit-test-result-action@v2 +# if: always() +# with: +# files: | +# build/test-output/*.xml # Then, we build and test using all licenses included proprietary ones build-properitary: name: Build & Test (Proprietary licenses) - needs: [ build-test ] +# needs: [ build-test, publish-test ] uses: ./.github/workflows/build-workflow.yml with: job_count: 4 @@ -47,7 +47,7 @@ jobs: upload_artifacts: false is_proprietary: true artifact_suffix: '-proprietary' - build_args: '-DHAS_GAMS=on -DGAMS_DIR=${GAMS_INSTALL_PATH} -DHAS_CPLEX=off -DHAS_GUROBI=on -DGUROBI_DIR=${GRB_INSTALL_PATH} -DCOMPILE_TESTS=on -DSPDLOG_STATIC=on' + build_args: '-DHAS_GAMS=on -DHAS_CPLEX=off -DHAS_GUROBI=on -DCOMPILE_TESTS=on -DSPDLOG_STATIC=on' secrets: gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} gams_license: ${{ secrets.GAMS_LICENSE_FILE }} diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index 44ae64ce..50f071f1 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -203,7 +203,7 @@ jobs: # Note the current convention is to use the -S and -B options here to specify source # and build directories, but this is only available with CMake 3.13 and higher. # The CMake binaries on the Github Actions machines are (as of this writing) 3.12 - run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DHAS_CBC=on -DCBC_DIR=${{github.workspace}}/ThirdParty/Cbc -DHAS_IPOPT=ON -DIPOPT_DIR=${{github.workspace}}/ThirdParty/Ipopt $EXTRA_BUILD_ARGS + run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DHAS_CBC=on -DCBC_DIR=${{github.workspace}}/ThirdParty/Cbc -DHAS_IPOPT=ON -DIPOPT_DIR=${{github.workspace}}/ThirdParty/Ipopt -DGAMS_DIR=$GAMS_INSTALL_PATH -DGUROBI_DIR=$GRB_INSTALL_PATH $EXTRA_BUILD_ARGS - name: Build working-directory: ${{github.workspace}}/build From e673aedc42aa4472450528a73b82682c22761881 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Wed, 22 Mar 2023 16:27:14 +0200 Subject: [PATCH 033/111] chore(ci): Supress output from wget to not clog the logs --- .github/workflows/build-workflow.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index 50f071f1..f83559cf 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -157,7 +157,7 @@ jobs: if: ${{ inputs.is_proprietary }} shell: bash working-directory: ${{github.workspace}}/ThirdParty - run: wget https://packages.gurobi.com/10.0/gurobi10.0.1_linux64.tar.gz ; tar -xvf gurobi10.0.1_linux64.tar.gz ; rm gurobi10.0.1_linux64.tar.gz + run: wget -q https://packages.gurobi.com/10.0/gurobi10.0.1_linux64.tar.gz ; tar -xvf gurobi10.0.1_linux64.tar.gz ; rm gurobi10.0.1_linux64.tar.gz # Finally, we set the environment variables needed for Gurobi to work. - name: Setup Gurobi environment variables @@ -175,7 +175,7 @@ jobs: if: ${{ inputs.is_proprietary }} shell: bash working-directory: ${{github.workspace}}/ThirdParty - run: wget https://d37drm4t2jghv5.cloudfront.net/distributions/42.3.0/linux/linux_x64_64_sfx.exe ; chmod +x linux_x64_64_sfx.exe ; ./linux_x64_64_sfx.exe ; rm linux_x64_64_sfx.exe + run: wget -q https://d37drm4t2jghv5.cloudfront.net/distributions/42.3.0/linux/linux_x64_64_sfx.exe ; chmod +x linux_x64_64_sfx.exe ; ./linux_x64_64_sfx.exe ; rm linux_x64_64_sfx.exe # Add the license file to the GAMS installation directory - name: Setup GAMS License file From 2d42f840431ece81abf187b2edd8a9d158b910d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Wed, 22 Mar 2023 16:31:44 +0200 Subject: [PATCH 034/111] refactor(ci): Run the proprietary build after publishing the test results and re-enable the normal build --- .github/workflows/build-push.yml | 56 ++++++++++++++++---------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index e906cab1..26e1b84b 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -7,38 +7,38 @@ on: [ push ] concurrency: push-build jobs: -# # First, we build and test using the default build options and only open source external libraries -# build-test: -# name: Build & Test -# uses: ./.github/workflows/build-workflow.yml -# with: -# job_count: 4 -# build_type: Release -# retention: 30 -# upload_artifacts: true -# -# # Publish the test results as an output -# publish-test: -# name: Publish test results -# needs: build-test -# runs-on: [ self-hosted, docker ] -# steps: -# - uses: actions/download-artifact@v3 -# name: Download test results -# with: -# name: ${{ needs.build-test.outputs.test-artifact }} -# -# - name: Publish Test Results -# uses: EnricoMi/publish-unit-test-result-action@v2 -# if: always() -# with: -# files: | -# build/test-output/*.xml + # First, we build and test using the default build options and only open source external libraries + build-test: + name: Build & Test + uses: ./.github/workflows/build-workflow.yml + with: + job_count: 4 + build_type: Release + retention: 30 + upload_artifacts: true + + # Publish the test results as an output + publish-test: + name: Publish test results + needs: build-test + runs-on: [ self-hosted, docker ] + steps: + - uses: actions/download-artifact@v3 + name: Download test results + with: + name: ${{ needs.build-test.outputs.test-artifact }} + + - name: Publish Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + files: | + build/test-output/*.xml # Then, we build and test using all licenses included proprietary ones build-properitary: name: Build & Test (Proprietary licenses) -# needs: [ build-test, publish-test ] + needs: [ build-test, publish-test ] uses: ./.github/workflows/build-workflow.yml with: job_count: 4 From 700407cfc59e3cc2d7a486678d085d9eb0d74b96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Wed, 22 Mar 2023 16:40:50 +0200 Subject: [PATCH 035/111] refactor(ci): Run the publish action last in an attempt to make the publish action publish all artifacts --- .github/workflows/build-push.yml | 41 ++++++++++++++++---------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 26e1b84b..9ad6983d 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -17,28 +17,10 @@ jobs: retention: 30 upload_artifacts: true - # Publish the test results as an output - publish-test: - name: Publish test results - needs: build-test - runs-on: [ self-hosted, docker ] - steps: - - uses: actions/download-artifact@v3 - name: Download test results - with: - name: ${{ needs.build-test.outputs.test-artifact }} - - - name: Publish Test Results - uses: EnricoMi/publish-unit-test-result-action@v2 - if: always() - with: - files: | - build/test-output/*.xml - # Then, we build and test using all licenses included proprietary ones - build-properitary: + build-proprietary: name: Build & Test (Proprietary licenses) - needs: [ build-test, publish-test ] + needs: [ build-test ] uses: ./.github/workflows/build-workflow.yml with: job_count: 4 @@ -52,3 +34,22 @@ jobs: gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} gams_license: ${{ secrets.GAMS_LICENSE_FILE }} + + # Publish the test results as an output + publish-test: + name: Publish test results + needs: [ build-test, build-proprietary ] + runs-on: [ self-hosted, docker ] + steps: + - uses: actions/download-artifact@v3 + name: Download artifacts + with: + path: artifacts + + - name: Publish Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + files: | + artifacts/build/test-output/*.xml + From 7a1a3efb5df1ed5aa222b58ab752aef4eac792f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Wed, 22 Mar 2023 17:03:24 +0200 Subject: [PATCH 036/111] fix(ci): Use the correct artifact naming/download format --- .github/workflows/build-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 9ad6983d..d072aa65 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -51,5 +51,5 @@ jobs: if: always() with: files: | - artifacts/build/test-output/*.xml + artifacts/test-*/*.xml From 802cf90c14af5d88919cae59e87f96293381cffe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Thu, 27 Apr 2023 13:11:42 +0300 Subject: [PATCH 037/111] refactor(ci): Refactored the proprietary switch to enable gams and gurobi separate of each-other. --- .github/workflows/build-push.yml | 4 +-- .github/workflows/build-workflow.yml | 39 +++++++++++++++++++++------- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index d072aa65..3d52d410 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -27,9 +27,9 @@ jobs: build_type: Release retention: 30 upload_artifacts: false - is_proprietary: true + enable_gurobi: true + enable_gams: true artifact_suffix: '-proprietary' - build_args: '-DHAS_GAMS=on -DHAS_CPLEX=off -DHAS_GUROBI=on -DCOMPILE_TESTS=on -DSPDLOG_STATIC=on' secrets: gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} gams_license: ${{ secrets.GAMS_LICENSE_FILE }} diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index f83559cf..c3b2baf2 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -34,8 +34,13 @@ on: default: true required: false type: boolean - is_proprietary: - description: 'Is this a proprietary build (this will install Gurobi and GAMS)' + enable_gurobi: + description: 'Is this a Gurobi build (this will install Gurobi)' + default: false + required: false + type: boolean + enable_gams: + description: 'Is this a GAMS build (this will install GAMS)' default: false required: false type: boolean @@ -92,6 +97,10 @@ jobs: name: Build SHOT runs-on: [ self-hosted, linux, cmake ] + env: + GAMS_OPTIONS: -DHAS_GAMS=on + GUROBI_OPTIONS: -DHAS_GUROBI=on + steps: - name: Cleanup workspace run: | @@ -147,21 +156,21 @@ jobs: # Proprietary software setup happens here (GAMS and Gurobi) # First we create the license files from the base64 encoded secrets. - name: Setup Gurobi License file - if: ${{ inputs.is_proprietary }} + if: ${{ inputs.enable_gurobi }} shell: bash working-directory: ${{github.workspace}} run: echo ${{ secrets.gurobi_license }} | base64 -d > $GRB_LICENSE_FILE # Then we download and install Gurobi - name: Download Gurobi - if: ${{ inputs.is_proprietary }} + if: ${{ inputs.enable_gurobi }} shell: bash working-directory: ${{github.workspace}}/ThirdParty run: wget -q https://packages.gurobi.com/10.0/gurobi10.0.1_linux64.tar.gz ; tar -xvf gurobi10.0.1_linux64.tar.gz ; rm gurobi10.0.1_linux64.tar.gz # Finally, we set the environment variables needed for Gurobi to work. - name: Setup Gurobi environment variables - if: ${{ inputs.is_proprietary }} + if: ${{ inputs.enable_gurobi }} shell: bash working-directory: ${{github.workspace}}/ThirdParty run: | @@ -169,32 +178,42 @@ jobs: echo "GUROBI_HOME=${{ github.workspace }}/ThirdParty/gurobi1001/linux64" >> $GITHUB_ENV echo "LD_LIBRARY_PATH=${{ github.workspace }}/ThirdParty/gurobi1001/linux64/lib" >> $GITHUB_ENV + - name: Set Gurobi options + if: ${{ inputs.enable_gurobi }} + shell: bash + run: echo "GUROBI_OPTIONS=-DHAS_GUROBI=on" >> $GITHUB_ENV + # Next, we set up GAMS # Download GAMS - name: Download GAMS - if: ${{ inputs.is_proprietary }} + if: ${{ inputs.enable_gams }} shell: bash working-directory: ${{github.workspace}}/ThirdParty run: wget -q https://d37drm4t2jghv5.cloudfront.net/distributions/42.3.0/linux/linux_x64_64_sfx.exe ; chmod +x linux_x64_64_sfx.exe ; ./linux_x64_64_sfx.exe ; rm linux_x64_64_sfx.exe # Add the license file to the GAMS installation directory - name: Setup GAMS License file - if: ${{ inputs.is_proprietary }} + if: ${{ inputs.enable_gams }} shell: bash working-directory: ${{github.workspace}} run: echo ${{ secrets.gams_license }} | base64 -d > $GAMS_LICENSE_FILE - name: Install GAMS - if: ${{ inputs.is_proprietary }} + if: ${{ inputs.enable_gams }} shell: bash working-directory: ${{ github.workspace }}/ThirdParty/gams42.3_linux_x64_64_sfx/ run: ./gamsinst -a - name: Add GAMS to the path - if: ${{ inputs.is_proprietary }} + if: ${{ inputs.enable_gams }} shell: bash run: echo "$GAMS_INSTALL_PATH" >> $GITHUB_PATH + - name: Set GAMS options + if: ${{ inputs.enable_gams }} + shell: bash + run: echo "GAMS_OPTIONS=-DHAS_GAMS=on" >> $GITHUB_ENV + - name: Configure CMake # Use a bash shell so we can use the same syntax for environment variable # access regardless of the host operating system @@ -203,7 +222,7 @@ jobs: # Note the current convention is to use the -S and -B options here to specify source # and build directories, but this is only available with CMake 3.13 and higher. # The CMake binaries on the Github Actions machines are (as of this writing) 3.12 - run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DHAS_CBC=on -DCBC_DIR=${{github.workspace}}/ThirdParty/Cbc -DHAS_IPOPT=ON -DIPOPT_DIR=${{github.workspace}}/ThirdParty/Ipopt -DGAMS_DIR=$GAMS_INSTALL_PATH -DGUROBI_DIR=$GRB_INSTALL_PATH $EXTRA_BUILD_ARGS + run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DHAS_CBC=on -DCBC_DIR=${{github.workspace}}/ThirdParty/Cbc -DHAS_IPOPT=ON -DIPOPT_DIR=${{github.workspace}}/ThirdParty/Ipopt -DGAMS_DIR=$GAMS_INSTALL_PATH -DGUROBI_DIR=$GRB_INSTALL_PATH -DCOMPILE_TESTS=on -DSPDLOG_STATIC=on $GAMS_OPTIONS $GUROBI_OPTIONS -DHAS_CPLEX=off - name: Build working-directory: ${{github.workspace}}/build From ac5ef674d883da036b8db30add64c7147c740073 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Thu, 27 Apr 2023 17:56:31 +0300 Subject: [PATCH 038/111] fix(ci): do not try to use gurobi/gams by default --- .github/workflows/build-workflow.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index c3b2baf2..0f434904 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -98,8 +98,8 @@ jobs: runs-on: [ self-hosted, linux, cmake ] env: - GAMS_OPTIONS: -DHAS_GAMS=on - GUROBI_OPTIONS: -DHAS_GUROBI=on + GAMS_OPTIONS: -DHAS_GAMS=off + GUROBI_OPTIONS: -DHAS_GUROBI=off steps: - name: Cleanup workspace From 00248938f0876251d03a3ac32317ce887e40ff1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sun, 24 Sep 2023 15:43:19 +0300 Subject: [PATCH 039/111] feat(ci): Allow for running the default workflow manually --- .github/workflows/build-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 3d52d410..937b13f1 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -1,7 +1,7 @@ name: Build # Build this on push -on: [ push ] +on: [ push, workflow_dispatch ] # Only build one build at a time concurrency: push-build From a487deafec0828a3c005a54c9254f49d70bbc463 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sun, 24 Sep 2023 17:58:01 +0300 Subject: [PATCH 040/111] feat(ci): Added the benchmarking step --- .github/workflows/build-push.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 937b13f1..6fe3ef55 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -34,6 +34,18 @@ jobs: gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} gams_license: ${{ secrets.GAMS_LICENSE_FILE }} + # Finally, we run some simple benchmarks + benchmark: + name: Benchmark + needs: [ build-test, build-proprietary ] + runs-on: [ self-hosted, docker ] + steps: + - name: Run Benchmark + uses: maxemiliang/shot-benchmarker@v1 + with: + benchmark_folder: "MINLP-convex-small" + benchmark_type: "nl" + shot_executable: "shot" # Publish the test results as an output publish-test: From 2828609ac800328552a0e50b3dd8037e7ef78254 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sun, 24 Sep 2023 18:02:26 +0300 Subject: [PATCH 041/111] fix(ci): Updated the SHOT executable path. --- .github/workflows/build-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 6fe3ef55..91ebbc6f 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -45,7 +45,7 @@ jobs: with: benchmark_folder: "MINLP-convex-small" benchmark_type: "nl" - shot_executable: "shot" + shot_executable: ${{github.workspace}}/build/SHOT # Publish the test results as an output publish-test: From 30ceaace62ec966e0c5af9d2c9fd6f4c2bba9427 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sun, 24 Sep 2023 18:48:08 +0300 Subject: [PATCH 042/111] fix(ci): Updated the version for the action --- .github/workflows/build-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 91ebbc6f..1afbe785 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -41,7 +41,7 @@ jobs: runs-on: [ self-hosted, docker ] steps: - name: Run Benchmark - uses: maxemiliang/shot-benchmarker@v1 + uses: maxemiliang/shot-benchmarker@v1.1 with: benchmark_folder: "MINLP-convex-small" benchmark_type: "nl" From fc2a29696c1df6ce66476f3c19d653e9526fc2d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sun, 1 Oct 2023 18:03:13 +0300 Subject: [PATCH 043/111] fix(ci): Updated the version for the action --- .github/workflows/build-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 1afbe785..2804ecf5 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -41,7 +41,7 @@ jobs: runs-on: [ self-hosted, docker ] steps: - name: Run Benchmark - uses: maxemiliang/shot-benchmarker@v1.1 + uses: maxemiliang/shot-benchmarker@v1.2 with: benchmark_folder: "MINLP-convex-small" benchmark_type: "nl" From df3b1f4fdfc3b52f4a972460bae4fcd8883e74b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Tue, 10 Oct 2023 21:24:58 +0300 Subject: [PATCH 044/111] fix(ci): updated the action version to -> 1.3 --- .github/workflows/build-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 2804ecf5..4a6b308f 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -41,7 +41,7 @@ jobs: runs-on: [ self-hosted, docker ] steps: - name: Run Benchmark - uses: maxemiliang/shot-benchmarker@v1.2 + uses: maxemiliang/shot-benchmarker@v1.3 with: benchmark_folder: "MINLP-convex-small" benchmark_type: "nl" From b3304dc71969d81b75aed8a62282a570177d0161 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Tue, 10 Oct 2023 22:18:14 +0300 Subject: [PATCH 045/111] fix(ci): updated the action version to -> 1.4 and disabled proprietary build for a while --- .github/workflows/build-push.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 4a6b308f..ed3e3adf 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -18,21 +18,21 @@ jobs: upload_artifacts: true # Then, we build and test using all licenses included proprietary ones - build-proprietary: - name: Build & Test (Proprietary licenses) - needs: [ build-test ] - uses: ./.github/workflows/build-workflow.yml - with: - job_count: 4 - build_type: Release - retention: 30 - upload_artifacts: false - enable_gurobi: true - enable_gams: true - artifact_suffix: '-proprietary' - secrets: - gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} - gams_license: ${{ secrets.GAMS_LICENSE_FILE }} +# build-proprietary: +# name: Build & Test (Proprietary licenses) +# needs: [ build-test ] +# uses: ./.github/workflows/build-workflow.yml +# with: +# job_count: 4 +# build_type: Release +# retention: 30 +# upload_artifacts: false +# enable_gurobi: true +# enable_gams: true +# artifact_suffix: '-proprietary' +# secrets: +# gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} +# gams_license: ${{ secrets.GAMS_LICENSE_FILE }} # Finally, we run some simple benchmarks benchmark: @@ -41,7 +41,7 @@ jobs: runs-on: [ self-hosted, docker ] steps: - name: Run Benchmark - uses: maxemiliang/shot-benchmarker@v1.3 + uses: maxemiliang/shot-benchmarker@v1.4 with: benchmark_folder: "MINLP-convex-small" benchmark_type: "nl" From 2485be0b22b8fce6efbfadd318188b2a114cdcec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Tue, 10 Oct 2023 22:19:39 +0300 Subject: [PATCH 046/111] fix(ci): Fixed an issue with the commented step. --- .github/workflows/build-push.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index ed3e3adf..acf883bd 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -37,7 +37,7 @@ jobs: # Finally, we run some simple benchmarks benchmark: name: Benchmark - needs: [ build-test, build-proprietary ] + needs: [ build-test ] runs-on: [ self-hosted, docker ] steps: - name: Run Benchmark @@ -50,7 +50,7 @@ jobs: # Publish the test results as an output publish-test: name: Publish test results - needs: [ build-test, build-proprietary ] + needs: [ build-test ] runs-on: [ self-hosted, docker ] steps: - uses: actions/download-artifact@v3 From 22ac5df58a39194cbb16236ef422cc475a320ddd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Tue, 10 Oct 2023 22:39:35 +0300 Subject: [PATCH 047/111] fix(ci): updated the action version to -> 1.5 --- .github/workflows/build-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index acf883bd..e929a8f3 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -41,7 +41,7 @@ jobs: runs-on: [ self-hosted, docker ] steps: - name: Run Benchmark - uses: maxemiliang/shot-benchmarker@v1.4 + uses: maxemiliang/shot-benchmarker@v1.5 with: benchmark_folder: "MINLP-convex-small" benchmark_type: "nl" From f991c5564b2f7b80ec250d53a3f44ed7c6ed1a4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Tue, 17 Oct 2023 20:59:42 +0300 Subject: [PATCH 048/111] chore: Updated the version of the benchmarker --- .github/workflows/build-push.yml | 2 +- .gitignore | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index e929a8f3..59f799e7 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -41,7 +41,7 @@ jobs: runs-on: [ self-hosted, docker ] steps: - name: Run Benchmark - uses: maxemiliang/shot-benchmarker@v1.5 + uses: maxemiliang/shot-benchmarker@v1.6 with: benchmark_folder: "MINLP-convex-small" benchmark_type: "nl" diff --git a/.gitignore b/.gitignore index 58c4171e..40a14ae0 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ build/* /.vs /out/build +/.cache/ \ No newline at end of file From 2a5516f08dae099e5b21c7777cd470b4b06adf39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Tue, 17 Oct 2023 21:02:59 +0300 Subject: [PATCH 049/111] fix: temporarily remove the work folder as sudo --- .github/workflows/build-workflow.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index 0f434904..47961a43 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -104,8 +104,10 @@ jobs: steps: - name: Cleanup workspace run: | - rm -rf ./* || true - rm -rf ./.??* || true + sudo rm -rf ./* || true + rm -rf ./* || true + 3 + sudo rm -rf ./.??* || true - uses: actions/checkout@v3 From 0bb341130c5af352d81692b1007be6e57959e8a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Tue, 17 Oct 2023 21:04:07 +0300 Subject: [PATCH 050/111] fix: malformed previous commit --- .github/workflows/build-workflow.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index 47961a43..5463c28c 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -105,8 +105,6 @@ jobs: - name: Cleanup workspace run: | sudo rm -rf ./* || true - rm -rf ./* || true - 3 sudo rm -rf ./.??* || true - uses: actions/checkout@v3 From eeea793c9ae919bf8e85e9519b3280e63a180efa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Tue, 17 Oct 2023 21:39:35 +0300 Subject: [PATCH 051/111] chore: updated action version -> 1.7 --- .github/workflows/build-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 59f799e7..6849e743 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -41,7 +41,7 @@ jobs: runs-on: [ self-hosted, docker ] steps: - name: Run Benchmark - uses: maxemiliang/shot-benchmarker@v1.6 + uses: maxemiliang/shot-benchmarker@v1.7 with: benchmark_folder: "MINLP-convex-small" benchmark_type: "nl" From 751fbddfb9e54af2382c98aa671a1d410c8ff0c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Wed, 18 Oct 2023 23:20:33 +0300 Subject: [PATCH 052/111] chore: bump composite action version --- .github/workflows/build-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 6849e743..2516252d 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -41,7 +41,7 @@ jobs: runs-on: [ self-hosted, docker ] steps: - name: Run Benchmark - uses: maxemiliang/shot-benchmarker@v1.7 + uses: maxemiliang/shot-benchmarker@v1.8 with: benchmark_folder: "MINLP-convex-small" benchmark_type: "nl" From ad91fdb4cda76070b7230f258fd43b4bef3277b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Wed, 18 Oct 2023 23:40:39 +0300 Subject: [PATCH 053/111] refactor: move the benchmark action into this repo --- .github/workflows/benchmarker.yml | 54 +++++++++++++++++++++++++++++++ .github/workflows/build-push.yml | 13 +++----- 2 files changed, 59 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/benchmarker.yml diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml new file mode 100644 index 00000000..4b8d975b --- /dev/null +++ b/.github/workflows/benchmarker.yml @@ -0,0 +1,54 @@ +name: Benchmark + +on: + workflow_call: + inputs: + shot_executable: + description: 'The SHOT executable to run' + default: "shot" + required: false + type: string + benchmark_folder: + description: 'The benchmark to run (folder name from: https://github.com/andreaslundell/SHOT_benchmark_problems)' + default: "MINLP-convex-small" + required: false + type: string + benchmark_type: + description: 'The benchmark type to run, can be gms, nl or osil' + default: "nl" + required: false + type: string + benchmarks: + description: 'The benchmarks to run, can be a list of benchmarks or a single benchmark, defaults to all benchmarks' + default: "" + required: false + type: string + +jobs: + benchmark: + name: Benchmark SHOT + runs-on: [ self-hosted, linux, cmake ] + + steps: + - shell: bash + run: git clone https://github.com/maxemiliang/shot-benchmarker.git + + + - uses: actions/setup-python@v4 + with: + python-version: 3.11 + cache: 'pip' + working-directory: ${{ github.workspace }}/shot-benchmarker + + - shell: bash + run: pip install -r requirements.txt + working-directory: ${{ github.workspace }}/shot-benchmarker + + - shell: bash + env: + INPUT_BENCHMARK_FOLDER: ${{ inputs.benchmark_folder }} + INPUT_BENCHMARK_TYPE: ${{ inputs.benchmark_type }} + INPUT_BENCHMARKS: ${{ inputs.benchmarks }} + INPUT_SHOT_EXECUTABLE: ${{ inputs.shot_executable }} + run: python main.py + working-directory: ${{ github.workspace }}/shot-benchmarker \ No newline at end of file diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 2516252d..fcae6288 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -38,14 +38,11 @@ jobs: benchmark: name: Benchmark needs: [ build-test ] - runs-on: [ self-hosted, docker ] - steps: - - name: Run Benchmark - uses: maxemiliang/shot-benchmarker@v1.8 - with: - benchmark_folder: "MINLP-convex-small" - benchmark_type: "nl" - shot_executable: ${{github.workspace}}/build/SHOT + uses: ./.github/workflows/benchmarker.yml§ + with: + benchmark_folder: "MINLP-convex-small" + benchmark_type: "nl" + shot_executable: ${{github.workspace}}/build/SHOT # Publish the test results as an output publish-test: From 1c9edf667730f7a39bc8c2bacf240faa37ce2c4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Wed, 18 Oct 2023 23:41:32 +0300 Subject: [PATCH 054/111] fix: typo fix --- .github/workflows/build-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index fcae6288..d04ea9d2 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -38,7 +38,7 @@ jobs: benchmark: name: Benchmark needs: [ build-test ] - uses: ./.github/workflows/benchmarker.yml§ + uses: ./.github/workflows/benchmarker.yml with: benchmark_folder: "MINLP-convex-small" benchmark_type: "nl" From 6f030b8d9b3e6615a39cce7df596587c834d07f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Thu, 19 Oct 2023 00:01:16 +0300 Subject: [PATCH 055/111] fix: Fixed an issue with working-directory indentation --- .github/workflows/benchmarker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index 4b8d975b..6d26d153 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -38,7 +38,7 @@ jobs: with: python-version: 3.11 cache: 'pip' - working-directory: ${{ github.workspace }}/shot-benchmarker + working-directory: ${{ github.workspace }}/shot-benchmarker - shell: bash run: pip install -r requirements.txt @@ -50,5 +50,5 @@ jobs: INPUT_BENCHMARK_TYPE: ${{ inputs.benchmark_type }} INPUT_BENCHMARKS: ${{ inputs.benchmarks }} INPUT_SHOT_EXECUTABLE: ${{ inputs.shot_executable }} + working-directory: ${{ github.workspace }}/shot-benchmarker run: python main.py - working-directory: ${{ github.workspace }}/shot-benchmarker \ No newline at end of file From e0e56476183d24164596184eb902d7a7e8d901ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Thu, 19 Oct 2023 00:13:23 +0300 Subject: [PATCH 056/111] fix: Move the working directory for the python script correctly --- .github/workflows/benchmarker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index 6d26d153..52a6c7f9 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -50,5 +50,5 @@ jobs: INPUT_BENCHMARK_TYPE: ${{ inputs.benchmark_type }} INPUT_BENCHMARKS: ${{ inputs.benchmarks }} INPUT_SHOT_EXECUTABLE: ${{ inputs.shot_executable }} - working-directory: ${{ github.workspace }}/shot-benchmarker + working-directory: ${{ github.workspace }}/shot-benchmarker run: python main.py From 06ba4db8c900cdbd6b3f0113ff919fb993d18d3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Thu, 19 Oct 2023 22:39:15 +0300 Subject: [PATCH 057/111] fix: some minor fixes to the benchmarker workflow --- .github/workflows/benchmarker.yml | 5 ++++- .github/workflows/build-push.yml | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index 52a6c7f9..4bda3a94 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -20,7 +20,7 @@ on: type: string benchmarks: description: 'The benchmarks to run, can be a list of benchmarks or a single benchmark, defaults to all benchmarks' - default: "" + default: "all" required: false type: string @@ -44,6 +44,9 @@ jobs: run: pip install -r requirements.txt working-directory: ${{ github.workspace }}/shot-benchmarker + - shell: bash + run: echo ${{ inputs.shot_executable }} + - shell: bash env: INPUT_BENCHMARK_FOLDER: ${{ inputs.benchmark_folder }} diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index d04ea9d2..d6a65a1d 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -42,7 +42,7 @@ jobs: with: benchmark_folder: "MINLP-convex-small" benchmark_type: "nl" - shot_executable: ${{github.workspace}}/build/SHOT + shot_executable: ${{ github.workspace }}/build/SHOT # Publish the test results as an output publish-test: From 7a07e3fa8a0f4034e001cd4d742fb34f1d837e30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Thu, 19 Oct 2023 22:55:21 +0300 Subject: [PATCH 058/111] chore: comment out more steps to ease debugging --- .github/workflows/build-push.yml | 50 ++++++++++++++++---------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index d6a65a1d..cefa564a 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -8,14 +8,14 @@ concurrency: push-build jobs: # First, we build and test using the default build options and only open source external libraries - build-test: - name: Build & Test - uses: ./.github/workflows/build-workflow.yml - with: - job_count: 4 - build_type: Release - retention: 30 - upload_artifacts: true +# build-test: +# name: Build & Test +# uses: ./.github/workflows/build-workflow.yml +# with: +# job_count: 4 +# build_type: Release +# retention: 30 +# upload_artifacts: true # Then, we build and test using all licenses included proprietary ones # build-proprietary: @@ -37,7 +37,7 @@ jobs: # Finally, we run some simple benchmarks benchmark: name: Benchmark - needs: [ build-test ] +# needs: [ build-test ] uses: ./.github/workflows/benchmarker.yml with: benchmark_folder: "MINLP-convex-small" @@ -45,20 +45,20 @@ jobs: shot_executable: ${{ github.workspace }}/build/SHOT # Publish the test results as an output - publish-test: - name: Publish test results - needs: [ build-test ] - runs-on: [ self-hosted, docker ] - steps: - - uses: actions/download-artifact@v3 - name: Download artifacts - with: - path: artifacts - - - name: Publish Test Results - uses: EnricoMi/publish-unit-test-result-action@v2 - if: always() - with: - files: | - artifacts/test-*/*.xml +# publish-test: +# name: Publish test results +# needs: [ build-test ] +# runs-on: [ self-hosted, docker ] +# steps: +# - uses: actions/download-artifact@v3 +# name: Download artifacts +# with: +# path: artifacts +# +# - name: Publish Test Results +# uses: EnricoMi/publish-unit-test-result-action@v2 +# if: always() +# with: +# files: | +# artifacts/test-*/*.xml From 3e56f00cbdca614ec4f5cd16611953766e60e0d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Thu, 19 Oct 2023 22:59:58 +0300 Subject: [PATCH 059/111] fix: Set the default where github context apparently is available. --- .github/workflows/benchmarker.yml | 2 +- .github/workflows/build-push.yml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index 4bda3a94..a99ab797 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -5,7 +5,7 @@ on: inputs: shot_executable: description: 'The SHOT executable to run' - default: "shot" + default: ${{ github.workspace }}/build/SHOT required: false type: string benchmark_folder: diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index cefa564a..a28fb298 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -42,7 +42,6 @@ jobs: with: benchmark_folder: "MINLP-convex-small" benchmark_type: "nl" - shot_executable: ${{ github.workspace }}/build/SHOT # Publish the test results as an output # publish-test: From 6c4c3f5bb6978321c958bea0939bfbaf7cff081e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Thu, 19 Oct 2023 23:02:25 +0300 Subject: [PATCH 060/111] fix: use the env variable instead --- .github/workflows/benchmarker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index a99ab797..54d208b4 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -5,7 +5,7 @@ on: inputs: shot_executable: description: 'The SHOT executable to run' - default: ${{ github.workspace }}/build/SHOT + default: ${{ env.GITHUB_WORKSPACE }}/build/SHOT required: false type: string benchmark_folder: From c2dd029c3433500b9c7dfa77e4c093914a76f556 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Thu, 19 Oct 2023 23:04:41 +0300 Subject: [PATCH 061/111] fix: revert to context variable instead --- .github/workflows/benchmarker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index 54d208b4..a99ab797 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -5,7 +5,7 @@ on: inputs: shot_executable: description: 'The SHOT executable to run' - default: ${{ env.GITHUB_WORKSPACE }}/build/SHOT + default: ${{ github.workspace }}/build/SHOT required: false type: string benchmark_folder: From 9203efd2a9f220fe34241d62608b5991469fc608 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Thu, 19 Oct 2023 23:09:24 +0300 Subject: [PATCH 062/111] fix: just use relative for now.. --- .github/workflows/benchmarker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index a99ab797..63b24b93 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -5,7 +5,7 @@ on: inputs: shot_executable: description: 'The SHOT executable to run' - default: ${{ github.workspace }}/build/SHOT + default: ./../build/SHOT required: false type: string benchmark_folder: From 2f22c63f4df347e6c53b2429e7260c0d21aeb107 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Thu, 19 Oct 2023 23:12:57 +0300 Subject: [PATCH 063/111] refactor: Make the git pull less error prone. --- .github/workflows/benchmarker.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index 63b24b93..e5e6550d 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -31,8 +31,7 @@ jobs: steps: - shell: bash - run: git clone https://github.com/maxemiliang/shot-benchmarker.git - + run: git pull -C shot-benchmarker || git clone https://github.com/maxemiliang/shot-benchmarker.git - uses: actions/setup-python@v4 with: From a37e4a25166a403489278c01d9a7c8582856ef9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Thu, 19 Oct 2023 23:14:00 +0300 Subject: [PATCH 064/111] refactor: Make the git pull less error prone. part 2 --- .github/workflows/benchmarker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index e5e6550d..bede7780 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -31,7 +31,7 @@ jobs: steps: - shell: bash - run: git pull -C shot-benchmarker || git clone https://github.com/maxemiliang/shot-benchmarker.git + run: git -C shot-benchmarker pull || git clone https://github.com/maxemiliang/shot-benchmarker.git - uses: actions/setup-python@v4 with: From 635b43a146d511d884d516e6c33f54ec78c0bd13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Thu, 19 Oct 2023 23:26:03 +0300 Subject: [PATCH 065/111] feat: Added the ran benchmarks as an output to the job. --- .github/workflows/benchmarker.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index bede7780..97dc63f6 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -24,8 +24,15 @@ on: required: false type: string + outputs: + benchmarks: + description: 'The benchmarks that were run' + value: ${{ jobs.benchmark.outputs.benchmarks }} + jobs: benchmark: + outputs: + benchmarks: ${{ steps.benchmark.outputs.benchmarks }} name: Benchmark SHOT runs-on: [ self-hosted, linux, cmake ] @@ -47,6 +54,7 @@ jobs: run: echo ${{ inputs.shot_executable }} - shell: bash + id: benchmark env: INPUT_BENCHMARK_FOLDER: ${{ inputs.benchmark_folder }} INPUT_BENCHMARK_TYPE: ${{ inputs.benchmark_type }} From 32e8e36d480cf19951f1c7f5531b48902ecde82b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Thu, 19 Oct 2023 23:47:22 +0300 Subject: [PATCH 066/111] feat: upload the benchmark osrl files --- .github/workflows/benchmarker.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index 97dc63f6..af99b57a 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -62,3 +62,9 @@ jobs: INPUT_SHOT_EXECUTABLE: ${{ inputs.shot_executable }} working-directory: ${{ github.workspace }}/shot-benchmarker run: python main.py + + - name: Upload benchmarks + uses: actions/upload-artifact@v3 + with: + name: benchmarks-${{ github.sha }} + path: ${{ github.workspace }}/shot-benchmarker/benchmarks From f9d844bd270df2748fd21707729ae69a9205d331 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Thu, 19 Oct 2023 23:49:42 +0300 Subject: [PATCH 067/111] chore: Added back the old build steps, to test the whole workflow --- .github/workflows/build-push.yml | 80 ++++++++++++++++---------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index a28fb298..2254f06d 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -8,56 +8,56 @@ concurrency: push-build jobs: # First, we build and test using the default build options and only open source external libraries -# build-test: -# name: Build & Test -# uses: ./.github/workflows/build-workflow.yml -# with: -# job_count: 4 -# build_type: Release -# retention: 30 -# upload_artifacts: true + build-test: + name: Build & Test + uses: ./.github/workflows/build-workflow.yml + with: + job_count: 4 + build_type: Release + retention: 30 + upload_artifacts: true # Then, we build and test using all licenses included proprietary ones -# build-proprietary: -# name: Build & Test (Proprietary licenses) -# needs: [ build-test ] -# uses: ./.github/workflows/build-workflow.yml -# with: -# job_count: 4 -# build_type: Release -# retention: 30 -# upload_artifacts: false -# enable_gurobi: true -# enable_gams: true -# artifact_suffix: '-proprietary' -# secrets: -# gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} -# gams_license: ${{ secrets.GAMS_LICENSE_FILE }} + build-proprietary: + name: Build & Test (Proprietary licenses) + needs: [ build-test ] + uses: ./.github/workflows/build-workflow.yml + with: + job_count: 4 + build_type: Release + retention: 30 + upload_artifacts: false + enable_gurobi: true + enable_gams: true + artifact_suffix: '-proprietary' + secrets: + gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} + gams_license: ${{ secrets.GAMS_LICENSE_FILE }} # Finally, we run some simple benchmarks benchmark: name: Benchmark -# needs: [ build-test ] + needs: [ build-test, build-proprietary ] uses: ./.github/workflows/benchmarker.yml with: benchmark_folder: "MINLP-convex-small" benchmark_type: "nl" # Publish the test results as an output -# publish-test: -# name: Publish test results -# needs: [ build-test ] -# runs-on: [ self-hosted, docker ] -# steps: -# - uses: actions/download-artifact@v3 -# name: Download artifacts -# with: -# path: artifacts -# -# - name: Publish Test Results -# uses: EnricoMi/publish-unit-test-result-action@v2 -# if: always() -# with: -# files: | -# artifacts/test-*/*.xml + publish-test: + name: Publish test results + needs: [ build-test, build-proprietary ] + runs-on: [ self-hosted, docker ] + steps: + - uses: actions/download-artifact@v3 + name: Download artifacts + with: + path: artifacts + + - name: Publish Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + files: | + artifacts/test-*/*.xml From fcd159153291b0e72b22014ae9d69a62d0a83b78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sun, 22 Oct 2023 21:42:07 +0300 Subject: [PATCH 068/111] refactor(ci): refactored the benchmarker to add GAMS/Gurobi license if it is needed --- .github/workflows/benchmarker.yml | 36 +++++++++++++++++++++++++++++++ .github/workflows/build-push.yml | 5 +++++ 2 files changed, 41 insertions(+) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index af99b57a..31905cc9 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -23,12 +23,35 @@ on: default: "all" required: false type: string + enable_gurobi: + description: 'Is this a Gurobi build (this will install Gurobi)' + default: false + required: false + type: boolean + enable_gams: + description: 'Is this a GAMS build (this will install GAMS)' + default: false + required: false + type: boolean + + secrets: + gurobi_license: + description: 'Gurobi license file secret' + required: false + gams_license: + description: 'GAMS license file secret' + required: false outputs: benchmarks: description: 'The benchmarks that were run' value: ${{ jobs.benchmark.outputs.benchmarks }} + +env: + GRB_LICENSE_FILE: ${{ github.workspace }}/gurobi.lic + GAMS_LICENSE_FILE: ${{ github.workspace }}/ThirdParty/gams42.3_linux_x64_64_sfx/gamslice.txt + jobs: benchmark: outputs: @@ -53,6 +76,19 @@ jobs: - shell: bash run: echo ${{ inputs.shot_executable }} + # Add the license file to the GAMS installation directory + - name: Setup GAMS License file + if: ${{ inputs.enable_gams }} + shell: bash + working-directory: ${{github.workspace}} + run: echo ${{ secrets.gams_license }} | base64 -d > $GAMS_LICENSE_FILE + + - name: Setup Gurobi License file + if: ${{ inputs.enable_gurobi }} + shell: bash + working-directory: ${{github.workspace}} + run: echo ${{ secrets.gurobi_license }} | base64 -d > $GRB_LICENSE_FILE + - shell: bash id: benchmark env: diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 2254f06d..1249e3a5 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -42,6 +42,11 @@ jobs: with: benchmark_folder: "MINLP-convex-small" benchmark_type: "nl" + enable_gurobi: true + enable_gams: true + secrets: + gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} + gams_license: ${{ secrets.GAMS_LICENSE_FILE }} # Publish the test results as an output publish-test: From af42c7815944d733f6397e68e85a023a81e4c559 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sun, 22 Oct 2023 22:08:44 +0300 Subject: [PATCH 069/111] fix(ci): Cleanup license files --- .github/workflows/benchmarker.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index 31905cc9..d9a99cc7 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -104,3 +104,10 @@ jobs: with: name: benchmarks-${{ github.sha }} path: ${{ github.workspace }}/shot-benchmarker/benchmarks + + - name: Cleanup license files + if: ${{ success() || failure() }} + shell: bash + run: | + rm -f $GRB_LICENSE_FILE + rm -f $GAMS_LICENSE_FILE From 35789d62690f70066c7efb0815fc6b930853d043 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Thu, 9 Nov 2023 20:59:11 +0200 Subject: [PATCH 070/111] refactor: benchmarker, test with artifact. --- .github/workflows/benchmarker.yml | 13 ++++++++++++- .github/workflows/build-push.yml | 5 +++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index d9a99cc7..b27ba205 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -33,7 +33,11 @@ on: default: false required: false type: boolean - + shot_artifact: + description: 'The SHOT artifact to use' + default: "" + required: false + type: string secrets: gurobi_license: description: 'Gurobi license file secret' @@ -60,6 +64,13 @@ jobs: runs-on: [ self-hosted, linux, cmake ] steps: + - name: Fetch SHOT artifact + if: ${{ inputs.shot_artifact != '' }} + uses: actions/download-artifact@v3 + with: + name: ${{ inputs.shot_artifact }} + path: ${{ github.workspace }}/build + - shell: bash run: git -C shot-benchmarker pull || git clone https://github.com/maxemiliang/shot-benchmarker.git diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 1249e3a5..5a66c30b 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -42,8 +42,9 @@ jobs: with: benchmark_folder: "MINLP-convex-small" benchmark_type: "nl" - enable_gurobi: true - enable_gams: true + enable_gurobi: false + enable_gams: false + shot_artifact: ${{ needs.build-test.outputs.binary-artifact }} secrets: gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} gams_license: ${{ secrets.GAMS_LICENSE_FILE }} From f1929eb92b16e22d410a86c9efde4932ac2610d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Thu, 9 Nov 2023 21:43:14 +0200 Subject: [PATCH 071/111] fix: attempt correct output of the artifacts name --- .github/workflows/build-push.yml | 34 +++++++++++++++++--------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 5a66c30b..2c03801b 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -16,23 +16,25 @@ jobs: build_type: Release retention: 30 upload_artifacts: true + outputs: + binary-artifact: ${{ steps.build.output-artifacts.binary }} - # Then, we build and test using all licenses included proprietary ones - build-proprietary: - name: Build & Test (Proprietary licenses) - needs: [ build-test ] - uses: ./.github/workflows/build-workflow.yml - with: - job_count: 4 - build_type: Release - retention: 30 - upload_artifacts: false - enable_gurobi: true - enable_gams: true - artifact_suffix: '-proprietary' - secrets: - gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} - gams_license: ${{ secrets.GAMS_LICENSE_FILE }} +# # Then, we build and test using all licenses included proprietary ones +# build-proprietary: +# name: Build & Test (Proprietary licenses) +# needs: [ build-test ] +# uses: ./.github/workflows/build-workflow.yml +# with: +# job_count: 4 +# build_type: Release +# retention: 30 +# upload_artifacts: false +# enable_gurobi: true +# enable_gams: true +# artifact_suffix: '-proprietary' +# secrets: +# gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} +# gams_license: ${{ secrets.GAMS_LICENSE_FILE }} # Finally, we run some simple benchmarks benchmark: From 3f51c666f66dcc2490f208841612f396784b80a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Thu, 9 Nov 2023 21:46:12 +0200 Subject: [PATCH 072/111] fix: move the outputs to the correct place and naming --- .github/workflows/build-push.yml | 2 -- .github/workflows/build-workflow.yml | 10 +++++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 2c03801b..b6d15a49 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -16,8 +16,6 @@ jobs: build_type: Release retention: 30 upload_artifacts: true - outputs: - binary-artifact: ${{ steps.build.output-artifacts.binary }} # # Then, we build and test using all licenses included proprietary ones # build-proprietary: diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index 5463c28c..10faf11c 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -89,11 +89,11 @@ env: jobs: build: outputs: - test-artifact: ${{ steps.artifact_outputs.test }} - options-artifact: ${{ steps.artifact_outputs.options }} - binary-artifact: ${{ steps.artifact_outputs.binary }} - package-artifact: ${{ steps.artifact_outputs.package }} - library-artifact: ${{ steps.artifact_outputs.library }} + test-artifact: ${{ steps.output-artifacts.test }} + options-artifact: ${{ steps.output-artifacts.options }} + binary-artifact: ${{ steps.output-artifacts.binary }} + package-artifact: ${{ steps.output-artifacts.package }} + library-artifact: ${{ steps.output-artifacts.library }} name: Build SHOT runs-on: [ self-hosted, linux, cmake ] From b91f2bef1f46ae1b5110c7b91c53bf08cb0a24a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Thu, 9 Nov 2023 21:48:07 +0200 Subject: [PATCH 073/111] fix: move the outputs to the correct place and naming --- .github/workflows/build-push.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index b6d15a49..6044e37b 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -18,21 +18,21 @@ jobs: upload_artifacts: true # # Then, we build and test using all licenses included proprietary ones -# build-proprietary: -# name: Build & Test (Proprietary licenses) -# needs: [ build-test ] -# uses: ./.github/workflows/build-workflow.yml -# with: -# job_count: 4 -# build_type: Release -# retention: 30 -# upload_artifacts: false -# enable_gurobi: true -# enable_gams: true -# artifact_suffix: '-proprietary' -# secrets: -# gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} -# gams_license: ${{ secrets.GAMS_LICENSE_FILE }} + build-proprietary: + name: Build & Test (Proprietary licenses) + needs: [ build-test ] + uses: ./.github/workflows/build-workflow.yml + with: + job_count: 4 + build_type: Release + retention: 30 + upload_artifacts: false + enable_gurobi: true + enable_gams: true + artifact_suffix: '-proprietary' + secrets: + gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} + gams_license: ${{ secrets.GAMS_LICENSE_FILE }} # Finally, we run some simple benchmarks benchmark: From a616ee57a04902205af8569e34bdddb304396d59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sat, 11 Nov 2023 18:37:52 +0200 Subject: [PATCH 074/111] =?UTF-8?q?fix:=20Attempt=20to=20just=20pass=20the?= =?UTF-8?q?=20name=20of=20the=20artifact=20directly.=20=E2=9C=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/benchmarker.yml | 5 ++--- .github/workflows/build-push.yml | 2 +- .github/workflows/build-workflow.yml | 10 +++++----- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index b27ba205..662b7cd7 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -78,7 +78,6 @@ jobs: with: python-version: 3.11 cache: 'pip' - working-directory: ${{ github.workspace }}/shot-benchmarker - shell: bash run: pip install -r requirements.txt @@ -91,13 +90,13 @@ jobs: - name: Setup GAMS License file if: ${{ inputs.enable_gams }} shell: bash - working-directory: ${{github.workspace}} + working-directory: ${{ github.workspace }} run: echo ${{ secrets.gams_license }} | base64 -d > $GAMS_LICENSE_FILE - name: Setup Gurobi License file if: ${{ inputs.enable_gurobi }} shell: bash - working-directory: ${{github.workspace}} + working-directory: ${{ github.workspace }} run: echo ${{ secrets.gurobi_license }} | base64 -d > $GRB_LICENSE_FILE - shell: bash diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 6044e37b..3eb00489 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -44,7 +44,7 @@ jobs: benchmark_type: "nl" enable_gurobi: false enable_gams: false - shot_artifact: ${{ needs.build-test.outputs.binary-artifact }} + shot_artifact: shot-binary-${{ github.sha }} secrets: gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} gams_license: ${{ secrets.GAMS_LICENSE_FILE }} diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index 10faf11c..c42415eb 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -300,11 +300,11 @@ jobs: if: ${{ inputs.upload_artifacts }} shell: bash run: | - echo "test=test-logs-${{ env.VERSION }}" >> $GITHUB_OUTPUT - echo "options=shot-optsfile-${{ env.VERSION }}" >> $GITHUB_OUTPUT - echo "binary=shot-binary-${{ env.VERSION }}" >> $GITHUB_OUTPUT - echo "package=shot-package-${{ env.VERSION }}" >> $GITHUB_OUTPUT - echo "library=shot-libraries-${{ env.VERSION }}" >> $GITHUB_OUTPUT + echo "test=test-logs-${{ env.VERSION }}${{ env.ARTIFACT_SUFFIX }}" >> $GITHUB_OUTPUT + echo "options=shot-optsfile-${{ env.VERSION }}${{ env.ARTIFACT_SUFFIX }}" >> $GITHUB_OUTPUT + echo "binary=shot-binary-${{ env.VERSION }}${{ env.ARTIFACT_SUFFIX }}" >> $GITHUB_OUTPUT + echo "package=shot-package-${{ env.VERSION }}${{ env.ARTIFACT_SUFFIX }}" >> $GITHUB_OUTPUT + echo "library=shot-libraries-${{ env.VERSION }}${{ env.ARTIFACT_SUFFIX }}" >> $GITHUB_OUTPUT - name: Output test artifacts if: ${{ inputs.upload_artifacts }} != true && steps.output-artifacts.outcome != 'success' From cf168ef323e4bc560b9fe169cf10052244228277 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Thu, 1 Feb 2024 19:17:04 +0200 Subject: [PATCH 075/111] chore(ci): Since we are using the latest version of CMake, we can use -S --- .github/workflows/build-workflow.yml | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index c42415eb..274207da 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -215,19 +215,20 @@ jobs: run: echo "GAMS_OPTIONS=-DHAS_GAMS=on" >> $GITHUB_ENV - name: Configure CMake - # Use a bash shell so we can use the same syntax for environment variable + # Uses bash shell, so we can use the same syntax for environment variable # access regardless of the host operating system shell: bash working-directory: ${{github.workspace}}/build - # Note the current convention is to use the -S and -B options here to specify source - # and build directories, but this is only available with CMake 3.13 and higher. - # The CMake binaries on the Github Actions machines are (as of this writing) 3.12 - run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DHAS_CBC=on -DCBC_DIR=${{github.workspace}}/ThirdParty/Cbc -DHAS_IPOPT=ON -DIPOPT_DIR=${{github.workspace}}/ThirdParty/Ipopt -DGAMS_DIR=$GAMS_INSTALL_PATH -DGUROBI_DIR=$GRB_INSTALL_PATH -DCOMPILE_TESTS=on -DSPDLOG_STATIC=on $GAMS_OPTIONS $GUROBI_OPTIONS -DHAS_CPLEX=off + # Here we configure the build, we set the build type, + # enable the tests and set the path to the external libraries. + run: cmake -S $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DHAS_CBC=on -DCBC_DIR=${{github.workspace}}/ThirdParty/Cbc -DHAS_IPOPT=ON -DIPOPT_DIR=${{github.workspace}}/ThirdParty/Ipopt -DGAMS_DIR=$GAMS_INSTALL_PATH -DGUROBI_DIR=$GRB_INSTALL_PATH -DCOMPILE_TESTS=on -DSPDLOG_STATIC=on $GAMS_OPTIONS $GUROBI_OPTIONS -DHAS_CPLEX=off - name: Build working-directory: ${{github.workspace}}/build shell: bash - # Execute the build. You can specify a specific target with "--target " + # We build the job, $BUILD_TYPE can be Release, Debug, RelWithDebInfo. + # $JOB_COUNT is the amount of jobs to use when building the program and dependencies, + # this can be set as an input. run: cmake --build . --config $BUILD_TYPE -j $JOB_COUNT - name: Generate Optionsfile @@ -239,15 +240,11 @@ jobs: id: test working-directory: ${{github.workspace}}/build shell: bash - # Execute tests defined by the CMake configuration. - # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail run: ctest -C $BUILD_TYPE --output-on-failure --output-junit test-output/ctest-junit-output.xml --output-log test-output/ctest-test.log - name: Construct package working-directory: ${{github.workspace}}/build shell: bash - # Execute tests defined by the CMake configuration. - # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail run: make package # This will always run as we always want the test output, however if the files are not found, we just warn the user. From 3fd73b196c6d9e18414e9d259fe71ce6580f8e99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sun, 17 Mar 2024 21:13:26 +0200 Subject: [PATCH 076/111] feat(ci): Updated the benchmarker + added secrets. Also temporarly disable the licensed builds. --- .github/workflows/benchmarker.yml | 11 ++++++++++- .github/workflows/build-push.yml | 32 +++++++++++++++---------------- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index 662b7cd7..cd216d8b 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -106,8 +106,17 @@ jobs: INPUT_BENCHMARK_TYPE: ${{ inputs.benchmark_type }} INPUT_BENCHMARKS: ${{ inputs.benchmarks }} INPUT_SHOT_EXECUTABLE: ${{ inputs.shot_executable }} + OS_PROJECT_ID: ${{ secrets.OS_PROJECT_ID }} + OS_PROJECT_NAME: ${{ secrets.OS_PROJECT_NAME }} + OS_USER_DOMAIN_NAME: "Default" + OS_PROJECT_DOMAIN_ID: "default" + OS_USERNAME: ${{ secrets.OS_USERNAME }} + OS_PASSWORD: ${{ secrets.OS_PASSWORD }} + OS_REGION_NAME: "regionOne" + OS_INTERFACE: "public" + OS_IDENTITY_API_VERSION: "3" working-directory: ${{ github.workspace }}/shot-benchmarker - run: python main.py + run: python main.py -c -s - name: Upload benchmarks uses: actions/upload-artifact@v3 diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 3eb00489..e41950d5 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -17,22 +17,22 @@ jobs: retention: 30 upload_artifacts: true -# # Then, we build and test using all licenses included proprietary ones - build-proprietary: - name: Build & Test (Proprietary licenses) - needs: [ build-test ] - uses: ./.github/workflows/build-workflow.yml - with: - job_count: 4 - build_type: Release - retention: 30 - upload_artifacts: false - enable_gurobi: true - enable_gams: true - artifact_suffix: '-proprietary' - secrets: - gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} - gams_license: ${{ secrets.GAMS_LICENSE_FILE }} + # Then, we build and test using all licenses included proprietary ones +# build-proprietary: +# name: Build & Test (Proprietary licenses) +# needs: [ build-test ] +# uses: ./.github/workflows/build-workflow.yml +# with: +# job_count: 4 +# build_type: Release +# retention: 30 +# upload_artifacts: false +# enable_gurobi: true +# enable_gams: true +# artifact_suffix: '-proprietary' +# secrets: +# gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} +# gams_license: ${{ secrets.GAMS_LICENSE_FILE }} # Finally, we run some simple benchmarks benchmark: From 21effd9f6a1a1fcd3a263623cb5381d9ab98e610 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sun, 17 Mar 2024 21:14:35 +0200 Subject: [PATCH 077/111] fix(ci): Fixed an issue with the build dependencies as the licensed build is not running --- .github/workflows/build-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index e41950d5..6092b6ea 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -52,7 +52,7 @@ jobs: # Publish the test results as an output publish-test: name: Publish test results - needs: [ build-test, build-proprietary ] + needs: [ build-test ] runs-on: [ self-hosted, docker ] steps: - uses: actions/download-artifact@v3 From 4e06b647fabc41d7e3330408c0422bbf9f41342e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sun, 17 Mar 2024 21:14:35 +0200 Subject: [PATCH 078/111] fix(ci): Fixed an issue with the build dependencies as the licensed build is not running --- .github/workflows/build-push.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index e41950d5..5ba5d43c 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -37,7 +37,7 @@ jobs: # Finally, we run some simple benchmarks benchmark: name: Benchmark - needs: [ build-test, build-proprietary ] + needs: [ build-test ] uses: ./.github/workflows/benchmarker.yml with: benchmark_folder: "MINLP-convex-small" @@ -52,7 +52,7 @@ jobs: # Publish the test results as an output publish-test: name: Publish test results - needs: [ build-test, build-proprietary ] + needs: [ build-test ] runs-on: [ self-hosted, docker ] steps: - uses: actions/download-artifact@v3 From b74e9e12013fbc09a2b6a2e3f6d1af1b0a8b8c01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sun, 17 Mar 2024 21:37:30 +0200 Subject: [PATCH 079/111] fix(ci): Added Auth URL secret --- .github/workflows/benchmarker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index cd216d8b..2aca69be 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -106,6 +106,7 @@ jobs: INPUT_BENCHMARK_TYPE: ${{ inputs.benchmark_type }} INPUT_BENCHMARKS: ${{ inputs.benchmarks }} INPUT_SHOT_EXECUTABLE: ${{ inputs.shot_executable }} + OS_AUTH_URL: ${{ secrets.OS_AUTH_URL }} OS_PROJECT_ID: ${{ secrets.OS_PROJECT_ID }} OS_PROJECT_NAME: ${{ secrets.OS_PROJECT_NAME }} OS_USER_DOMAIN_NAME: "Default" From 9b941f6b8dd6f574361edb6e24c9f3e24420175a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sun, 17 Mar 2024 21:52:06 +0200 Subject: [PATCH 080/111] fix(ci): Added GITHUB_TOKEN secret --- .github/workflows/benchmarker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index 2aca69be..b08bc89a 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -116,6 +116,7 @@ jobs: OS_REGION_NAME: "regionOne" OS_INTERFACE: "public" OS_IDENTITY_API_VERSION: "3" + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} working-directory: ${{ github.workspace }}/shot-benchmarker run: python main.py -c -s From b22eca247cc8986be66df8f7e5a9cbe6e55a1f70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sun, 17 Mar 2024 22:12:55 +0200 Subject: [PATCH 081/111] fix(ci): Correctly wrap the values to escape special characters. --- .github/workflows/benchmarker.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index b08bc89a..01f9bdb1 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -106,17 +106,17 @@ jobs: INPUT_BENCHMARK_TYPE: ${{ inputs.benchmark_type }} INPUT_BENCHMARKS: ${{ inputs.benchmarks }} INPUT_SHOT_EXECUTABLE: ${{ inputs.shot_executable }} - OS_AUTH_URL: ${{ secrets.OS_AUTH_URL }} - OS_PROJECT_ID: ${{ secrets.OS_PROJECT_ID }} - OS_PROJECT_NAME: ${{ secrets.OS_PROJECT_NAME }} + OS_AUTH_URL: '${{ secrets.OS_AUTH_URL }}' + OS_PROJECT_ID: '${{ secrets.OS_PROJECT_ID }}' + OS_PROJECT_NAME: '${{ secrets.OS_PROJECT_NAME }}' OS_USER_DOMAIN_NAME: "Default" OS_PROJECT_DOMAIN_ID: "default" - OS_USERNAME: ${{ secrets.OS_USERNAME }} - OS_PASSWORD: ${{ secrets.OS_PASSWORD }} + OS_USERNAME: '${{ secrets.OS_USERNAME }}' + OS_PASSWORD: '${{ secrets.OS_PASSWORD }}' OS_REGION_NAME: "regionOne" OS_INTERFACE: "public" OS_IDENTITY_API_VERSION: "3" - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}' working-directory: ${{ github.workspace }}/shot-benchmarker run: python main.py -c -s From 08b50c772f2a23af46c2479ac244665c46dcc479 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sun, 17 Mar 2024 23:08:49 +0200 Subject: [PATCH 082/111] fix(ci): Disable all other builds + some testing stuff --- .github/workflows/benchmarker.yml | 6 ++-- .github/workflows/build-push.yml | 52 +++++++++++++++---------------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index 01f9bdb1..275e8cb9 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -106,9 +106,9 @@ jobs: INPUT_BENCHMARK_TYPE: ${{ inputs.benchmark_type }} INPUT_BENCHMARKS: ${{ inputs.benchmarks }} INPUT_SHOT_EXECUTABLE: ${{ inputs.shot_executable }} - OS_AUTH_URL: '${{ secrets.OS_AUTH_URL }}' - OS_PROJECT_ID: '${{ secrets.OS_PROJECT_ID }}' - OS_PROJECT_NAME: '${{ secrets.OS_PROJECT_NAME }}' + OS_AUTH_URL: 'https://pouta.csc.fi:5001/v3' + OS_PROJECT_ID: '230dabcfb234424cb77e7de501ef7efc' + OS_PROJECT_NAME: 'project_2001223' OS_USER_DOMAIN_NAME: "Default" OS_PROJECT_DOMAIN_ID: "default" OS_USERNAME: '${{ secrets.OS_USERNAME }}' diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 5ba5d43c..0530d6b6 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -8,14 +8,14 @@ concurrency: push-build jobs: # First, we build and test using the default build options and only open source external libraries - build-test: - name: Build & Test - uses: ./.github/workflows/build-workflow.yml - with: - job_count: 4 - build_type: Release - retention: 30 - upload_artifacts: true +# build-test: +# name: Build & Test +# uses: ./.github/workflows/build-workflow.yml +# with: +# job_count: 4 +# build_type: Release +# retention: 30 +# upload_artifacts: true # Then, we build and test using all licenses included proprietary ones # build-proprietary: @@ -37,33 +37,33 @@ jobs: # Finally, we run some simple benchmarks benchmark: name: Benchmark - needs: [ build-test ] +# needs: [ build-test ] uses: ./.github/workflows/benchmarker.yml with: benchmark_folder: "MINLP-convex-small" benchmark_type: "nl" enable_gurobi: false enable_gams: false - shot_artifact: shot-binary-${{ github.sha }} + shot_artifact: shot-binary-b22eca247cc8986be66df8f7e5a9cbe6e55a1f70 secrets: gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} gams_license: ${{ secrets.GAMS_LICENSE_FILE }} # Publish the test results as an output - publish-test: - name: Publish test results - needs: [ build-test ] - runs-on: [ self-hosted, docker ] - steps: - - uses: actions/download-artifact@v3 - name: Download artifacts - with: - path: artifacts - - - name: Publish Test Results - uses: EnricoMi/publish-unit-test-result-action@v2 - if: always() - with: - files: | - artifacts/test-*/*.xml +# publish-test: +# name: Publish test results +# needs: [ build-test ] +# runs-on: [ self-hosted, docker ] +# steps: +# - uses: actions/download-artifact@v3 +# name: Download artifacts +# with: +# path: artifacts +# +# - name: Publish Test Results +# uses: EnricoMi/publish-unit-test-result-action@v2 +# if: always() +# with: +# files: | +# artifacts/test-*/*.xml From de804a4a6099ee0296974aa95df1e1e3cb968d56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sun, 17 Mar 2024 23:10:38 +0200 Subject: [PATCH 083/111] fix(ci): Re-enable building --- .github/workflows/build-push.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 0530d6b6..37437ff6 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -8,14 +8,14 @@ concurrency: push-build jobs: # First, we build and test using the default build options and only open source external libraries -# build-test: -# name: Build & Test -# uses: ./.github/workflows/build-workflow.yml -# with: -# job_count: 4 -# build_type: Release -# retention: 30 -# upload_artifacts: true + build-test: + name: Build & Test + uses: ./.github/workflows/build-workflow.yml + with: + job_count: 4 + build_type: Release + retention: 30 + upload_artifacts: true # Then, we build and test using all licenses included proprietary ones # build-proprietary: @@ -37,14 +37,14 @@ jobs: # Finally, we run some simple benchmarks benchmark: name: Benchmark -# needs: [ build-test ] + needs: [ build-test ] uses: ./.github/workflows/benchmarker.yml with: benchmark_folder: "MINLP-convex-small" benchmark_type: "nl" enable_gurobi: false enable_gams: false - shot_artifact: shot-binary-b22eca247cc8986be66df8f7e5a9cbe6e55a1f70 + shot_artifact: shot-binary-${{ github.sha }} secrets: gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} gams_license: ${{ secrets.GAMS_LICENSE_FILE }} From 34523960dfc1c98d73b24ac259dfe7791639b2d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sun, 17 Mar 2024 23:35:16 +0200 Subject: [PATCH 084/111] fix(ci): Removed unnecessary step --- .github/workflows/benchmarker.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index 275e8cb9..9c32749e 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -83,9 +83,6 @@ jobs: run: pip install -r requirements.txt working-directory: ${{ github.workspace }}/shot-benchmarker - - shell: bash - run: echo ${{ inputs.shot_executable }} - # Add the license file to the GAMS installation directory - name: Setup GAMS License file if: ${{ inputs.enable_gams }} @@ -111,8 +108,8 @@ jobs: OS_PROJECT_NAME: 'project_2001223' OS_USER_DOMAIN_NAME: "Default" OS_PROJECT_DOMAIN_ID: "default" - OS_USERNAME: '${{ secrets.OS_USERNAME }}' - OS_PASSWORD: '${{ secrets.OS_PASSWORD }}' + OS_USERNAME: ${{ secrets.OS_USERNAME }} + OS_PASSWORD: ${{ secrets.OS_PASSWORD }} OS_REGION_NAME: "regionOne" OS_INTERFACE: "public" OS_IDENTITY_API_VERSION: "3" From ed3967a7af4d9ff362826b7878d34ade72f4d517 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sun, 17 Mar 2024 23:47:15 +0200 Subject: [PATCH 085/111] fix(ci): Actually pass the secrets to the reusable workflow.. --- .github/workflows/benchmarker.yml | 7 +++++++ .github/workflows/build-push.yml | 2 ++ 2 files changed, 9 insertions(+) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index 9c32749e..01951d0e 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -45,6 +45,13 @@ on: gams_license: description: 'GAMS license file secret' required: false + OS_USERNAME: + description: 'OpenStack username' + required: true + OS_PASSWORD: + description: 'OpenStack password' + required: true + outputs: benchmarks: diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 37437ff6..3afb30be 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -48,6 +48,8 @@ jobs: secrets: gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} gams_license: ${{ secrets.GAMS_LICENSE_FILE }} + OS_USERNAME: ${{ secrets.OS_USERNAME }} + OS_PASSWORD: ${{ secrets.OS_PASSWORD }} # Publish the test results as an output # publish-test: From 4ed607d42f2cbb8bee43645df0941959eade4123 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Mon, 18 Mar 2024 00:02:58 +0200 Subject: [PATCH 086/111] fix(ci): Wrap the secret to escape --- .github/workflows/benchmarker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index 01951d0e..01f08451 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -115,8 +115,8 @@ jobs: OS_PROJECT_NAME: 'project_2001223' OS_USER_DOMAIN_NAME: "Default" OS_PROJECT_DOMAIN_ID: "default" - OS_USERNAME: ${{ secrets.OS_USERNAME }} - OS_PASSWORD: ${{ secrets.OS_PASSWORD }} + OS_USERNAME: '${{ secrets.OS_USERNAME }}' + OS_PASSWORD: '${{ secrets.OS_PASSWORD }}' OS_REGION_NAME: "regionOne" OS_INTERFACE: "public" OS_IDENTITY_API_VERSION: "3" From de28256c08ed039e1de3bb38ba4707858dd1fbde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Fri, 29 Mar 2024 16:20:17 +0200 Subject: [PATCH 087/111] chore: add gams/gurobi status to the benchmarker --- .github/workflows/benchmarker.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index 01f08451..49a77a40 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -110,6 +110,8 @@ jobs: INPUT_BENCHMARK_TYPE: ${{ inputs.benchmark_type }} INPUT_BENCHMARKS: ${{ inputs.benchmarks }} INPUT_SHOT_EXECUTABLE: ${{ inputs.shot_executable }} + INPUT_IS_GAMS: ${{ inputs.enable_gams }} + INPUT_IS_GUROBI: ${{ inputs.enable_gurobi }} OS_AUTH_URL: 'https://pouta.csc.fi:5001/v3' OS_PROJECT_ID: '230dabcfb234424cb77e7de501ef7efc' OS_PROJECT_NAME: 'project_2001223' From 8151310a7e6af484d9e6b833dbd6214fcedc768f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sat, 6 Apr 2024 22:25:19 +0300 Subject: [PATCH 088/111] refactor: Allow for different Gurobi and GAMS versions. Also test the new Benchmarker. --- .github/workflows/build-workflow.yml | 46 ++++++++++++++++++++++------ misc/runner-init.yml | 1 + 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index 274207da..182476b8 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -49,6 +49,16 @@ on: default: '' required: false type: string + gurobi_version: + description: 'The version of Gurobi to use' + default: '11.0.1' + required: false + type: string + gams_version: + description: 'The version of GAMS to use' + default: '46.4.0' + required: false + type: string secrets: gurobi_license: description: 'Gurobi license file secret' @@ -82,9 +92,8 @@ env: ARTIFACT_SUFFIX: ${{ inputs.artifact_suffix }} EXTRA_BUILD_ARGS: ${{ inputs.build_args }} GRB_LICENSE_FILE: ${{ github.workspace }}/gurobi.lic - GRB_INSTALL_PATH: ${{ github.workspace }}/ThirdParty/gurobi1001 - GAMS_LICENSE_FILE: ${{ github.workspace }}/ThirdParty/gams42.3_linux_x64_64_sfx/gamslice.txt - GAMS_INSTALL_PATH: ${{ github.workspace }}/ThirdParty/gams42.3_linux_x64_64_sfx + GUROBI_VERSION: ${{ inputs.gurobi_version }} + GAMS_VERSION: ${{ inputs.gams_version }} jobs: build: @@ -161,12 +170,22 @@ jobs: working-directory: ${{github.workspace}} run: echo ${{ secrets.gurobi_license }} | base64 -d > $GRB_LICENSE_FILE + - name: Extract Version + id: get_version + run: | + version=${{ env.GUROBI_VERSION }} + major_minor=$(echo ${version%%.*}) + version_without_dot=$(echo $version | tr -d .) + echo "GUROBI_SHORT_VERSION=$major_minor" >> $GITHUB_ENV + echo "GUROBI_VERSION_WITHOUT_DOT=$version_without_dot" >> $GITHUB_ENV + echo "GRB_INSTALL_PATH=${{ github.workspace }}/ThirdParty/gurobi${{ env.GUROBI_VERSION_WITHOUT_DOT }}" >> $GITHUB_ENV + # Then we download and install Gurobi - name: Download Gurobi if: ${{ inputs.enable_gurobi }} shell: bash working-directory: ${{github.workspace}}/ThirdParty - run: wget -q https://packages.gurobi.com/10.0/gurobi10.0.1_linux64.tar.gz ; tar -xvf gurobi10.0.1_linux64.tar.gz ; rm gurobi10.0.1_linux64.tar.gz + run: wget -q https://packages.gurobi.com/${{ env.GUROBI_SHORT_VERSION }}/gurobi${{ env.GUROBI_VERSION }}_linux64.tar.gz ; tar -xvf gurobi${{ env.GUROBI_VERSION }}_linux64.tar.gz ; rm gurobi${{ env.GUROBI_VERSION }}_linux64.tar.gz # Finally, we set the environment variables needed for Gurobi to work. - name: Setup Gurobi environment variables @@ -174,22 +193,31 @@ jobs: shell: bash working-directory: ${{github.workspace}}/ThirdParty run: | - echo "${{ github.workspace }}/ThirdParty/gurobi1001/linux64/bin" >> $GITHUB_PATH - echo "GUROBI_HOME=${{ github.workspace }}/ThirdParty/gurobi1001/linux64" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${{ github.workspace }}/ThirdParty/gurobi1001/linux64/lib" >> $GITHUB_ENV + echo "${{ github.workspace }}/ThirdParty/gurobi${{ env.GUROBI_VERSION_WITHOUT_DOT }}/linux64/bin" >> $GITHUB_PATH + echo "GUROBI_HOME=${{ github.workspace }}/ThirdParty/gurobi${{ env.GUROBI_VERSION_WITHOUT_DOT }}/linux64" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=${{ github.workspace }}/ThirdParty/gurobi${{ env.GUROBI_VERSION_WITHOUT_DOT }}/linux64/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV - name: Set Gurobi options if: ${{ inputs.enable_gurobi }} shell: bash run: echo "GUROBI_OPTIONS=-DHAS_GUROBI=on" >> $GITHUB_ENV + - name: Get GAMS short version + id: get_gams_short_version + run: | + version=${{ env.GAMS_VERSION }} + major_minor=$(echo ${version%%.*}) + echo "GAMS_SHORT_VERSION=$major_minor" >> $GITHUB_ENV + echo "GAMS_INSTALL_PATH=${{ github.workspace }}/ThirdParty/gams${{ env.GAMS_SHORT_VERSION }}_linux_x64_64_sfx" >> $GITHUB_ENV + echo "GAMS_LICENSE_FILE=${{ github.workspace }}/ThirdParty/gams${{ env.GAMS_SHORT_VERSION }}_linux_x64_64_sfx/gamslice.txt" >> $GITHUB_ENV + # Next, we set up GAMS # Download GAMS - name: Download GAMS if: ${{ inputs.enable_gams }} shell: bash working-directory: ${{github.workspace}}/ThirdParty - run: wget -q https://d37drm4t2jghv5.cloudfront.net/distributions/42.3.0/linux/linux_x64_64_sfx.exe ; chmod +x linux_x64_64_sfx.exe ; ./linux_x64_64_sfx.exe ; rm linux_x64_64_sfx.exe + run: wget -q https://d37drm4t2jghv5.cloudfront.net/distributions/${{ env.GAMS_VERSION }}/linux/linux_x64_64_sfx.exe ; chmod +x linux_x64_64_sfx.exe ; ./linux_x64_64_sfx.exe ; rm linux_x64_64_sfx.exe # Add the license file to the GAMS installation directory - name: Setup GAMS License file @@ -201,7 +229,7 @@ jobs: - name: Install GAMS if: ${{ inputs.enable_gams }} shell: bash - working-directory: ${{ github.workspace }}/ThirdParty/gams42.3_linux_x64_64_sfx/ + working-directory: ${{ github.workspace }}/ThirdParty/gams${{ env.GAMS_SHORT_VERSION }}_linux_x64_64_sfx/ run: ./gamsinst -a - name: Add GAMS to the path diff --git a/misc/runner-init.yml b/misc/runner-init.yml index 29d0fb1c..13468376 100644 --- a/misc/runner-init.yml +++ b/misc/runner-init.yml @@ -18,6 +18,7 @@ packages: - curl - gnupg - lsb-release + - ccache users: - default From aa5a59741aefe816a4233de3eb251de2ee5b09f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sat, 6 Apr 2024 22:46:18 +0300 Subject: [PATCH 089/111] feat: Re enable licensed build workflow --- .github/workflows/benchmarker.yml | 2 +- .github/workflows/build-push.yml | 66 +++++++++++++++---------------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index 49a77a40..f2cee1b4 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -124,7 +124,7 @@ jobs: OS_IDENTITY_API_VERSION: "3" GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}' working-directory: ${{ github.workspace }}/shot-benchmarker - run: python main.py -c -s + run: python main.py -c -s --runs 5 - name: Upload benchmarks uses: actions/upload-artifact@v3 diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 3afb30be..da166acb 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -18,26 +18,26 @@ jobs: upload_artifacts: true # Then, we build and test using all licenses included proprietary ones -# build-proprietary: -# name: Build & Test (Proprietary licenses) -# needs: [ build-test ] -# uses: ./.github/workflows/build-workflow.yml -# with: -# job_count: 4 -# build_type: Release -# retention: 30 -# upload_artifacts: false -# enable_gurobi: true -# enable_gams: true -# artifact_suffix: '-proprietary' -# secrets: -# gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} -# gams_license: ${{ secrets.GAMS_LICENSE_FILE }} + build-proprietary: + name: Build & Test (Proprietary licenses) + needs: [ build-test ] + uses: ./.github/workflows/build-workflow.yml + with: + job_count: 4 + build_type: Release + retention: 30 + upload_artifacts: false + enable_gurobi: true + enable_gams: true + artifact_suffix: '-proprietary' + secrets: + gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} + gams_license: ${{ secrets.GAMS_LICENSE_FILE }} # Finally, we run some simple benchmarks benchmark: name: Benchmark - needs: [ build-test ] + needs: [ build-test, build-proprietary ] uses: ./.github/workflows/benchmarker.yml with: benchmark_folder: "MINLP-convex-small" @@ -51,21 +51,21 @@ jobs: OS_USERNAME: ${{ secrets.OS_USERNAME }} OS_PASSWORD: ${{ secrets.OS_PASSWORD }} - # Publish the test results as an output -# publish-test: -# name: Publish test results -# needs: [ build-test ] -# runs-on: [ self-hosted, docker ] -# steps: -# - uses: actions/download-artifact@v3 -# name: Download artifacts -# with: -# path: artifacts -# -# - name: Publish Test Results -# uses: EnricoMi/publish-unit-test-result-action@v2 -# if: always() -# with: -# files: | -# artifacts/test-*/*.xml + # Publish the test results as an output + publish-test: + name: Publish test results + needs: [ build-test, build-proprietary, benchmark ] + runs-on: [ self-hosted, docker ] + steps: + - uses: actions/download-artifact@v3 + name: Download artifacts + with: + path: artifacts + + - name: Publish Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + files: | + artifacts/test-*/*.xml From 013ed532e8d45ef12b3d9ee0540549e0c31d8c90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sat, 6 Apr 2024 23:16:09 +0300 Subject: [PATCH 090/111] fix: Fixed an issue with getting the short version of GAMS and Gurobi from the workflow --- .github/workflows/build-workflow.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index 182476b8..9e79b62f 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -174,7 +174,7 @@ jobs: id: get_version run: | version=${{ env.GUROBI_VERSION }} - major_minor=$(echo ${version%%.*}) + major_minor=$(echo version | awk -F '.' '{print $1 "." $2}') version_without_dot=$(echo $version | tr -d .) echo "GUROBI_SHORT_VERSION=$major_minor" >> $GITHUB_ENV echo "GUROBI_VERSION_WITHOUT_DOT=$version_without_dot" >> $GITHUB_ENV @@ -206,7 +206,7 @@ jobs: id: get_gams_short_version run: | version=${{ env.GAMS_VERSION }} - major_minor=$(echo ${version%%.*}) + major_minor=$(echo version | awk -F '.' '{print $1 "." $2}') echo "GAMS_SHORT_VERSION=$major_minor" >> $GITHUB_ENV echo "GAMS_INSTALL_PATH=${{ github.workspace }}/ThirdParty/gams${{ env.GAMS_SHORT_VERSION }}_linux_x64_64_sfx" >> $GITHUB_ENV echo "GAMS_LICENSE_FILE=${{ github.workspace }}/ThirdParty/gams${{ env.GAMS_SHORT_VERSION }}_linux_x64_64_sfx/gamslice.txt" >> $GITHUB_ENV From 9b0dab0322e97cdc984479424e624d82be370c35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sat, 6 Apr 2024 23:23:29 +0300 Subject: [PATCH 091/111] fix: forgot to add $ to variable --- .github/workflows/build-workflow.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index 9e79b62f..07281c62 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -174,7 +174,7 @@ jobs: id: get_version run: | version=${{ env.GUROBI_VERSION }} - major_minor=$(echo version | awk -F '.' '{print $1 "." $2}') + major_minor=$(echo $version | awk -F '.' '{print $1 "." $2}') version_without_dot=$(echo $version | tr -d .) echo "GUROBI_SHORT_VERSION=$major_minor" >> $GITHUB_ENV echo "GUROBI_VERSION_WITHOUT_DOT=$version_without_dot" >> $GITHUB_ENV @@ -206,7 +206,7 @@ jobs: id: get_gams_short_version run: | version=${{ env.GAMS_VERSION }} - major_minor=$(echo version | awk -F '.' '{print $1 "." $2}') + major_minor=$(echo $version | awk -F '.' '{print $1 "." $2}') echo "GAMS_SHORT_VERSION=$major_minor" >> $GITHUB_ENV echo "GAMS_INSTALL_PATH=${{ github.workspace }}/ThirdParty/gams${{ env.GAMS_SHORT_VERSION }}_linux_x64_64_sfx" >> $GITHUB_ENV echo "GAMS_LICENSE_FILE=${{ github.workspace }}/ThirdParty/gams${{ env.GAMS_SHORT_VERSION }}_linux_x64_64_sfx/gamslice.txt" >> $GITHUB_ENV From 9c175f8b8deb49fe2da8095decb1e06be4206b9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sun, 7 Apr 2024 14:44:16 +0300 Subject: [PATCH 092/111] fix: Use the correct way to get the version to build the paths --- .github/workflows/build-workflow.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index 07281c62..8ca02c64 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -178,7 +178,7 @@ jobs: version_without_dot=$(echo $version | tr -d .) echo "GUROBI_SHORT_VERSION=$major_minor" >> $GITHUB_ENV echo "GUROBI_VERSION_WITHOUT_DOT=$version_without_dot" >> $GITHUB_ENV - echo "GRB_INSTALL_PATH=${{ github.workspace }}/ThirdParty/gurobi${{ env.GUROBI_VERSION_WITHOUT_DOT }}" >> $GITHUB_ENV + echo "GRB_INSTALL_PATH=${{ github.workspace }}/ThirdParty/gurobi$version_without_dot" >> $GITHUB_ENV # Then we download and install Gurobi - name: Download Gurobi @@ -208,8 +208,8 @@ jobs: version=${{ env.GAMS_VERSION }} major_minor=$(echo $version | awk -F '.' '{print $1 "." $2}') echo "GAMS_SHORT_VERSION=$major_minor" >> $GITHUB_ENV - echo "GAMS_INSTALL_PATH=${{ github.workspace }}/ThirdParty/gams${{ env.GAMS_SHORT_VERSION }}_linux_x64_64_sfx" >> $GITHUB_ENV - echo "GAMS_LICENSE_FILE=${{ github.workspace }}/ThirdParty/gams${{ env.GAMS_SHORT_VERSION }}_linux_x64_64_sfx/gamslice.txt" >> $GITHUB_ENV + echo "GAMS_INSTALL_PATH=${{ github.workspace }}/ThirdParty/gams$major_minor_linux_x64_64_sfx" >> $GITHUB_ENV + echo "GAMS_LICENSE_FILE=${{ github.workspace }}/ThirdParty/gams$major_minor_linux_x64_64_sfx/gamslice.txt" >> $GITHUB_ENV # Next, we set up GAMS # Download GAMS From b37251eac3eeabf6c9061bc6dd9a8e88bae34dc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sun, 7 Apr 2024 15:05:43 +0300 Subject: [PATCH 093/111] fix: bash variable expansion correctness --- .github/workflows/build-workflow.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index 8ca02c64..24627b21 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -178,7 +178,7 @@ jobs: version_without_dot=$(echo $version | tr -d .) echo "GUROBI_SHORT_VERSION=$major_minor" >> $GITHUB_ENV echo "GUROBI_VERSION_WITHOUT_DOT=$version_without_dot" >> $GITHUB_ENV - echo "GRB_INSTALL_PATH=${{ github.workspace }}/ThirdParty/gurobi$version_without_dot" >> $GITHUB_ENV + echo "GRB_INSTALL_PATH=${{ github.workspace }}/ThirdParty/gurobi${version_without_dot}" >> $GITHUB_ENV # Then we download and install Gurobi - name: Download Gurobi @@ -208,8 +208,8 @@ jobs: version=${{ env.GAMS_VERSION }} major_minor=$(echo $version | awk -F '.' '{print $1 "." $2}') echo "GAMS_SHORT_VERSION=$major_minor" >> $GITHUB_ENV - echo "GAMS_INSTALL_PATH=${{ github.workspace }}/ThirdParty/gams$major_minor_linux_x64_64_sfx" >> $GITHUB_ENV - echo "GAMS_LICENSE_FILE=${{ github.workspace }}/ThirdParty/gams$major_minor_linux_x64_64_sfx/gamslice.txt" >> $GITHUB_ENV + echo "GAMS_INSTALL_PATH=${{ github.workspace }}/ThirdParty/gams${major_minor}_linux_x64_64_sfx" >> $GITHUB_ENV + echo "GAMS_LICENSE_FILE=${{ github.workspace }}/ThirdParty/gams${major_minor}_linux_x64_64_sfx/gamslice.txt" >> $GITHUB_ENV # Next, we set up GAMS # Download GAMS From 947ab1312894188314f9418278da11dd7a2a73cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sun, 7 Apr 2024 15:14:38 +0300 Subject: [PATCH 094/111] fix: rollback to GAMS 42.3.0 --- .github/workflows/build-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index 24627b21..bf8f495c 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -56,7 +56,7 @@ on: type: string gams_version: description: 'The version of GAMS to use' - default: '46.4.0' + default: '42.3.0' required: false type: string secrets: From 75200e5bc035cc338e469e55050328f3e0895600 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sun, 7 Apr 2024 21:22:17 +0300 Subject: [PATCH 095/111] fix: moved shot binary to its own folder in the benchmark step to avoid conflicts. --- .github/workflows/benchmarker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index f2cee1b4..499a5e19 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -5,7 +5,7 @@ on: inputs: shot_executable: description: 'The SHOT executable to run' - default: ./../build/SHOT + default: ./../shot-binary/SHOT required: false type: string benchmark_folder: @@ -76,7 +76,7 @@ jobs: uses: actions/download-artifact@v3 with: name: ${{ inputs.shot_artifact }} - path: ${{ github.workspace }}/build + path: ${{ github.workspace }}/shot-binary - shell: bash run: git -C shot-benchmarker pull || git clone https://github.com/maxemiliang/shot-benchmarker.git From 531d8c3af52652c918c5f9808e9bcf9ed942bc19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sun, 7 Apr 2024 21:37:30 +0300 Subject: [PATCH 096/111] fix: More fixes to the benchmarker, to correctly handle permissions. --- .github/workflows/benchmarker.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index 499a5e19..993e3a0d 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -5,7 +5,7 @@ on: inputs: shot_executable: description: 'The SHOT executable to run' - default: ./../shot-binary/SHOT + default: ./SHOT required: false type: string benchmark_folder: @@ -71,15 +71,21 @@ jobs: runs-on: [ self-hosted, linux, cmake ] steps: + - shell: bash + run: git -C shot-benchmarker pull || git clone https://github.com/maxemiliang/shot-benchmarker.git + - name: Fetch SHOT artifact if: ${{ inputs.shot_artifact != '' }} uses: actions/download-artifact@v3 with: name: ${{ inputs.shot_artifact }} - path: ${{ github.workspace }}/shot-binary + path: ${{ github.workspace }}/shot-benchmarker - - shell: bash - run: git -C shot-benchmarker pull || git clone https://github.com/maxemiliang/shot-benchmarker.git + - name: Make SHOT executable + shell: bash + run: | + cd ${{ github.workspace }}/shot-benchmarker + chmod +x SHOT - uses: actions/setup-python@v4 with: From a2a0896bdc32cc4e6dd051af5ddf509d652a5d31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Wed, 17 Apr 2024 20:27:39 +0300 Subject: [PATCH 097/111] feat: cleanup workspace before benchmarking, to test if it helps with SHOT. --- .github/workflows/benchmarker.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index 993e3a0d..b7380e2d 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -71,6 +71,11 @@ jobs: runs-on: [ self-hosted, linux, cmake ] steps: + - name: Cleanup workspace + run: | + sudo rm -rf ./* || true + sudo rm -rf ./.??* || true + - shell: bash run: git -C shot-benchmarker pull || git clone https://github.com/maxemiliang/shot-benchmarker.git From 1845a58b41f767359a2b344ac20788a81d510f8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Wed, 17 Apr 2024 20:49:02 +0300 Subject: [PATCH 098/111] chore: Added linked libraries output to debug the build --- .github/workflows/build-workflow.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index bf8f495c..c6faa7d3 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -275,6 +275,11 @@ jobs: shell: bash run: make package + - name: Show linked libraries + working-directory: ${{github.workspace}}/build + shell: bash + run: ldd SHOT + # This will always run as we always want the test output, however if the files are not found, we just warn the user. - uses: actions/upload-artifact@v3 if: ${{ always() }} From 49b8a1b01aaf2ea41a5f56e1a2f0cc68163ef1bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Wed, 17 Apr 2024 20:57:26 +0300 Subject: [PATCH 099/111] chore: Added linked libraries output to debug the build --- .github/workflows/benchmarker.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index b7380e2d..5dd4b419 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -92,6 +92,11 @@ jobs: cd ${{ github.workspace }}/shot-benchmarker chmod +x SHOT + - name: Show linked libraries + shell: bash + working-directory: ${{ github.workspace }}/shot-benchmarker + run: ldd SHOT + - uses: actions/setup-python@v4 with: python-version: 3.11 From 9053d135bca9b889dcfa3d72d5a76dd1c0fd07bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sat, 20 Apr 2024 14:12:56 +0300 Subject: [PATCH 100/111] feat: Moved the benchmarking to be done directly after build, to correctly used the linked binaries. --- .github/workflows/benchmarker.yml | 20 +++++++++++++++----- .github/workflows/build-push.yml | 18 +----------------- .github/workflows/build-workflow.yml | 16 ++++++++++++++++ 3 files changed, 32 insertions(+), 22 deletions(-) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index 5dd4b419..e88fee18 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -3,9 +3,9 @@ name: Benchmark on: workflow_call: inputs: - shot_executable: - description: 'The SHOT executable to run' - default: ./SHOT + shot_executable_path: + description: 'The path to the SHOT executable to run' + default: "/SHOT" required: false type: string benchmark_folder: @@ -86,10 +86,20 @@ jobs: name: ${{ inputs.shot_artifact }} path: ${{ github.workspace }}/shot-benchmarker + - name: Set the SHOT executable path if an artifact is used + if : ${{ inputs.shot_executable_path == '' }} + shell: bash + run: echo "SHOT_EXECUTABLE_PATH=${{ github.workspace }}/shot-benchmarker" >> $GITHUB_ENV + + - name: Set the SHOT executable path + if: ${{ inputs.shot_executable_path != '' }} + shell: bash + run: echo "SHOT_EXECUTABLE_PATH=${{ inputs.shot_executable_path }}" >> $GITHUB_ENV + - name: Make SHOT executable shell: bash run: | - cd ${{ github.workspace }}/shot-benchmarker + cd ${{ env.SHOT_EXECUTABLE_PATH }} chmod +x SHOT - name: Show linked libraries @@ -125,7 +135,7 @@ jobs: INPUT_BENCHMARK_FOLDER: ${{ inputs.benchmark_folder }} INPUT_BENCHMARK_TYPE: ${{ inputs.benchmark_type }} INPUT_BENCHMARKS: ${{ inputs.benchmarks }} - INPUT_SHOT_EXECUTABLE: ${{ inputs.shot_executable }} + INPUT_SHOT_EXECUTABLE: ${{ env.SHOT_EXECUTABLE_PATH }}/SHOT INPUT_IS_GAMS: ${{ inputs.enable_gams }} INPUT_IS_GUROBI: ${{ inputs.enable_gurobi }} OS_AUTH_URL: 'https://pouta.csc.fi:5001/v3' diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index da166acb..537cd19e 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -34,24 +34,8 @@ jobs: gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} gams_license: ${{ secrets.GAMS_LICENSE_FILE }} - # Finally, we run some simple benchmarks - benchmark: - name: Benchmark - needs: [ build-test, build-proprietary ] - uses: ./.github/workflows/benchmarker.yml - with: - benchmark_folder: "MINLP-convex-small" - benchmark_type: "nl" - enable_gurobi: false - enable_gams: false - shot_artifact: shot-binary-${{ github.sha }} - secrets: - gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} - gams_license: ${{ secrets.GAMS_LICENSE_FILE }} - OS_USERNAME: ${{ secrets.OS_USERNAME }} - OS_PASSWORD: ${{ secrets.OS_PASSWORD }} - # Publish the test results as an output + # Publish the test results as an output publish-test: name: Publish test results needs: [ build-test, build-proprietary, benchmark ] diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index c6faa7d3..46cd72c5 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -347,3 +347,19 @@ jobs: run: | rm -f $GRB_LICENSE_FILE rm -f $GAMS_LICENSE_FILE + + # Finally, we run some simple benchmarks + benchmark: + name: Benchmark + uses: ./.github/workflows/benchmarker.yml + with: + shot_executable_path: "${{ github.workspace }}/build" + benchmark_folder: "MINLP-convex-small" + benchmark_type: "nl" + enable_gurobi: ${{ inputs.enable_gurobi }} + enable_gams: ${{ inputs.enable_gams }} + secrets: + gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} + gams_license: ${{ secrets.GAMS_LICENSE_FILE }} + OS_USERNAME: ${{ secrets.OS_USERNAME }} + OS_PASSWORD: ${{ secrets.OS_PASSWORD }} From a49fd1e98e8904fdf6b9efb14d36713321b5f29c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sat, 20 Apr 2024 14:13:40 +0300 Subject: [PATCH 101/111] fix: No longer depend on the benchmarker. --- .github/workflows/build-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 537cd19e..937b13f1 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -38,7 +38,7 @@ jobs: # Publish the test results as an output publish-test: name: Publish test results - needs: [ build-test, build-proprietary, benchmark ] + needs: [ build-test, build-proprietary ] runs-on: [ self-hosted, docker ] steps: - uses: actions/download-artifact@v3 From ed3d84e9eaa7a75f08124d09ee856a3cfa205128 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sat, 20 Apr 2024 14:16:46 +0300 Subject: [PATCH 102/111] fix: always run benchmarks after build + some inputs to the benchmarker and secrets. --- .github/workflows/build-push.yml | 6 ++++++ .github/workflows/build-workflow.yml | 13 +++++++++++++ 2 files changed, 19 insertions(+) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 937b13f1..395ffc5c 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -16,6 +16,10 @@ jobs: build_type: Release retention: 30 upload_artifacts: true + run_benchmarks: true + secrets: + OS_USERNAME: ${{ secrets.OS_USERNAME }} + OS_PASSWORD: ${{ secrets.OS_PASSWORD }} # Then, we build and test using all licenses included proprietary ones build-proprietary: @@ -33,6 +37,8 @@ jobs: secrets: gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} gams_license: ${{ secrets.GAMS_LICENSE_FILE }} + OS_USERNAME: ${{ secrets.OS_USERNAME }} + OS_PASSWORD: ${{ secrets.OS_PASSWORD }} # Publish the test results as an output diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index 46cd72c5..d4437871 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -59,6 +59,11 @@ on: default: '42.3.0' required: false type: string + run_benchmarks: + description: 'Should the benchmarks be run' + default: false + required: false + type: boolean secrets: gurobi_license: description: 'Gurobi license file secret' @@ -66,6 +71,12 @@ on: gams_license: description: 'GAMS license file secret' required: false + OS_USERNAME: + description: 'OpenStack username' + required: false + OS_PASSWORD: + description: 'OpenStack password' + required: false outputs: test-artifact: @@ -350,8 +361,10 @@ jobs: # Finally, we run some simple benchmarks benchmark: + if: ${{ inputs.run_benchmarks }} name: Benchmark uses: ./.github/workflows/benchmarker.yml + needs: [ build ] with: shot_executable_path: "${{ github.workspace }}/build" benchmark_folder: "MINLP-convex-small" From b278758443025f74937ffe957710e4c9ffd4864a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sat, 20 Apr 2024 14:21:21 +0300 Subject: [PATCH 103/111] fix: try to fix the usage of github workspace. --- .github/workflows/build-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index d4437871..2e597013 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -366,7 +366,7 @@ jobs: uses: ./.github/workflows/benchmarker.yml needs: [ build ] with: - shot_executable_path: "${{ github.workspace }}/build" + shot_executable_path: ${{ github.workspace }}/build benchmark_folder: "MINLP-convex-small" benchmark_type: "nl" enable_gurobi: ${{ inputs.enable_gurobi }} From 22447c92c3c99b969e869b18845426014aa6fdda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sat, 20 Apr 2024 14:23:37 +0300 Subject: [PATCH 104/111] fix: Updated the upload-artifacts workflow to use v4 --- .github/workflows/benchmarker.yml | 2 +- .github/workflows/build-workflow.yml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index e88fee18..d33e6360 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -153,7 +153,7 @@ jobs: run: python main.py -c -s --runs 5 - name: Upload benchmarks - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: benchmarks-${{ github.sha }} path: ${{ github.workspace }}/shot-benchmarker/benchmarks diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index 2e597013..e6cf4425 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -292,7 +292,7 @@ jobs: run: ldd SHOT # This will always run as we always want the test output, however if the files are not found, we just warn the user. - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: ${{ always() }} name: Upload test results with: @@ -302,7 +302,7 @@ jobs: ${{github.workspace}}/build/test-output/*.xml retention-days: ${{ inputs.retention }} - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: ${{ inputs.upload_artifacts }} name: Upload SHOT Optionsfile with: @@ -310,7 +310,7 @@ jobs: path: ${{github.workspace}}/build/SHOT.opt retention-days: ${{ inputs.retention }} - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: ${{ inputs.upload_artifacts }} name: Upload SHOT File with: @@ -318,7 +318,7 @@ jobs: path: ${{github.workspace}}/build/SHOT retention-days: ${{ inputs.retention }} - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: ${{ inputs.upload_artifacts }} name: Upload SHOT Package file with: @@ -326,7 +326,7 @@ jobs: path: ${{github.workspace}}/build/SHOT-*-Linux.zip retention-days: ${{ inputs.retention }} - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: ${{ inputs.upload_artifacts }} name: Upload SHOT Libraries with: From 13d70fe1792d22456117c3c048b0f1e5a2b694bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sat, 20 Apr 2024 14:29:52 +0300 Subject: [PATCH 105/111] fix: Fixed an issue with the benchmarker SHOT path --- .github/workflows/benchmarker.yml | 19 ++++++++++--------- .github/workflows/build-workflow.yml | 2 +- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index d33e6360..c9822114 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -71,17 +71,12 @@ jobs: runs-on: [ self-hosted, linux, cmake ] steps: - - name: Cleanup workspace - run: | - sudo rm -rf ./* || true - sudo rm -rf ./.??* || true - - shell: bash run: git -C shot-benchmarker pull || git clone https://github.com/maxemiliang/shot-benchmarker.git - name: Fetch SHOT artifact if: ${{ inputs.shot_artifact != '' }} - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: ${{ inputs.shot_artifact }} path: ${{ github.workspace }}/shot-benchmarker @@ -91,10 +86,16 @@ jobs: shell: bash run: echo "SHOT_EXECUTABLE_PATH=${{ github.workspace }}/shot-benchmarker" >> $GITHUB_ENV - - name: Set the SHOT executable path + # If the path is not an absolute path, we assume it is relative to the current working directory + - name: Set the SHOT executable path. if: ${{ inputs.shot_executable_path != '' }} shell: bash - run: echo "SHOT_EXECUTABLE_PATH=${{ inputs.shot_executable_path }}" >> $GITHUB_ENV + run: | + if [[ ${{ inputs.shot_executable_path }} == /* ]]; then + echo "SHOT_EXECUTABLE_PATH=${{ inputs.shot_executable_path }}" >> $GITHUB_ENV + else + echo "SHOT_EXECUTABLE_PATH=${{ github.workspace }}/${{ inputs.shot_executable_path }}" >> $GITHUB_ENV + fi - name: Make SHOT executable shell: bash @@ -104,7 +105,7 @@ jobs: - name: Show linked libraries shell: bash - working-directory: ${{ github.workspace }}/shot-benchmarker + working-directory: ${{ env.SHOT_EXECUTABLE_PATH }} run: ldd SHOT - uses: actions/setup-python@v4 diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index e6cf4425..920be175 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -366,7 +366,7 @@ jobs: uses: ./.github/workflows/benchmarker.yml needs: [ build ] with: - shot_executable_path: ${{ github.workspace }}/build + shot_executable_path: build benchmark_folder: "MINLP-convex-small" benchmark_type: "nl" enable_gurobi: ${{ inputs.enable_gurobi }} From aafcc59eded63c45d21410eee28ce6f2f832de12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sat, 20 Apr 2024 15:05:25 +0300 Subject: [PATCH 106/111] feat: Run proprietary license benchmarks on CI as well. --- .github/workflows/build-push.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 395ffc5c..a5334bde 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -34,6 +34,7 @@ jobs: enable_gurobi: true enable_gams: true artifact_suffix: '-proprietary' + run_benchmarks: true secrets: gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} gams_license: ${{ secrets.GAMS_LICENSE_FILE }} From 8ccdd22a2e1bb6819e322566216d011a25214e24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sat, 20 Apr 2024 15:06:20 +0300 Subject: [PATCH 107/111] fix: Download artifacts using the correct version of actions/download-artifact --- .github/workflows/build-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index a5334bde..78c55b10 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -48,7 +48,7 @@ jobs: needs: [ build-test, build-proprietary ] runs-on: [ self-hosted, docker ] steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 name: Download artifacts with: path: artifacts From c6046fd669f74ad8f4251a2d45b56f1e040f876f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sat, 20 Apr 2024 15:40:21 +0300 Subject: [PATCH 108/111] fix: Correctly set the license file path + license file contents in benchmarker --- .github/workflows/benchmarker.yml | 14 ++++++++++++-- .github/workflows/build-workflow.yml | 6 ++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index c9822114..529160ec 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -38,6 +38,16 @@ on: default: "" required: false type: string + gurobi_license_file: + description: 'The path to the Gurobi license file' + default: "" + required: false + type: string + gams_license_file: + description: 'The path to the GAMS license file' + default: "" + required: false + type: string secrets: gurobi_license: description: 'Gurobi license file secret' @@ -60,8 +70,8 @@ on: env: - GRB_LICENSE_FILE: ${{ github.workspace }}/gurobi.lic - GAMS_LICENSE_FILE: ${{ github.workspace }}/ThirdParty/gams42.3_linux_x64_64_sfx/gamslice.txt + GRB_LICENSE_FILE: ${{inputs.gurobi_license_file}} + GAMS_LICENSE_FILE: ${{inputs.gams_license_file}} jobs: benchmark: diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index 920be175..015937cb 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -371,8 +371,10 @@ jobs: benchmark_type: "nl" enable_gurobi: ${{ inputs.enable_gurobi }} enable_gams: ${{ inputs.enable_gams }} + gurobi_license_file: ${{ env.GUROBI_LICENSE_FILE }} + gams_license_file: ${{ env.GAMS_LICENSE_FILE }} secrets: - gurobi_license: ${{ secrets.GUROBI_LICENSE_FILE }} - gams_license: ${{ secrets.GAMS_LICENSE_FILE }} + gurobi_license: ${{ secrets.gurobi_license }} + gams_license: ${{ secrets.gams_license }} OS_USERNAME: ${{ secrets.OS_USERNAME }} OS_PASSWORD: ${{ secrets.OS_PASSWORD }} From c6c01b71cfbe3ba9f1cc23bbceb53925143b6690 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sat, 20 Apr 2024 19:06:41 +0300 Subject: [PATCH 109/111] fix: Corrected the license file path in the build workflow --- .github/workflows/build-workflow.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index 015937cb..62fc8266 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -114,6 +114,8 @@ jobs: binary-artifact: ${{ steps.output-artifacts.binary }} package-artifact: ${{ steps.output-artifacts.package }} library-artifact: ${{ steps.output-artifacts.library }} + gurobi_license_file: ${{ steps.licensefiles.outputs.gurobi}} + gams_license_file: ${{ steps.licensefiles.outputs.gams }} name: Build SHOT runs-on: [ self-hosted, linux, cmake ] @@ -253,6 +255,13 @@ jobs: shell: bash run: echo "GAMS_OPTIONS=-DHAS_GAMS=on" >> $GITHUB_ENV + - name: Output License files + id: licensefiles + shell: bash + run: | + echo "gurobi=${{ github.workspace }}/ThirdParty/gurobi${{ env.GUROBI_VERSION_WITHOUT_DOT }}/linux64/bin/gurobi.lic" >> $GITHUB_OUTPUT + echo "gams=${{ github.workspace }}/ThirdParty/gams${{ env.GAMS_SHORT_VERSION }}_linux_x64_64_sfx/gamslice.txt" >> $GITHUB_OUTPUT + - name: Configure CMake # Uses bash shell, so we can use the same syntax for environment variable # access regardless of the host operating system @@ -371,8 +380,8 @@ jobs: benchmark_type: "nl" enable_gurobi: ${{ inputs.enable_gurobi }} enable_gams: ${{ inputs.enable_gams }} - gurobi_license_file: ${{ env.GUROBI_LICENSE_FILE }} - gams_license_file: ${{ env.GAMS_LICENSE_FILE }} + gurobi_license_file: ${{ needs.build.outputs.gurobi_license_file }} + gams_license_file: ${{ needs.build.outputs.gams_license_file }} secrets: gurobi_license: ${{ secrets.gurobi_license }} gams_license: ${{ secrets.gams_license }} From 970dfe13131484c10df1b67d9ef5391d7bd15297 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sat, 20 Apr 2024 19:15:05 +0300 Subject: [PATCH 110/111] fix: Artifact suffix added to benchmarker workflow --- .github/workflows/benchmarker.yml | 7 ++++++- .github/workflows/build-workflow.yml | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index 529160ec..72e5e983 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -48,6 +48,11 @@ on: default: "" required: false type: string + artifact_suffix: + description: 'The suffix to add to the artifact name' + default: '' + required: false + type: string secrets: gurobi_license: description: 'Gurobi license file secret' @@ -166,7 +171,7 @@ jobs: - name: Upload benchmarks uses: actions/upload-artifact@v4 with: - name: benchmarks-${{ github.sha }} + name: benchmarks-${{ github.sha }}${{ inputs.artifact_suffix }} path: ${{ github.workspace }}/shot-benchmarker/benchmarks - name: Cleanup license files diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index 62fc8266..f5399291 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -382,6 +382,7 @@ jobs: enable_gams: ${{ inputs.enable_gams }} gurobi_license_file: ${{ needs.build.outputs.gurobi_license_file }} gams_license_file: ${{ needs.build.outputs.gams_license_file }} + artifact_suffix: ${{ inputs.artifact_suffix }} secrets: gurobi_license: ${{ secrets.gurobi_license }} gams_license: ${{ secrets.gams_license }} From 41297882fc4c4c720bfe0f26a3699579e499b8a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxemilian=20Gr=C3=B6nblom?= Date: Sat, 20 Apr 2024 19:38:31 +0300 Subject: [PATCH 111/111] refactor: benchmarker no longer compares Gurobi/Gams to Ipopt/CBC --- .github/workflows/benchmarker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/benchmarker.yml b/.github/workflows/benchmarker.yml index 72e5e983..a4831a5d 100644 --- a/.github/workflows/benchmarker.yml +++ b/.github/workflows/benchmarker.yml @@ -152,6 +152,7 @@ jobs: INPUT_BENCHMARK_TYPE: ${{ inputs.benchmark_type }} INPUT_BENCHMARKS: ${{ inputs.benchmarks }} INPUT_SHOT_EXECUTABLE: ${{ env.SHOT_EXECUTABLE_PATH }}/SHOT + INPUT_COMPARISON_SUFFIX: ${{ inputs.artifact_suffix }} INPUT_IS_GAMS: ${{ inputs.enable_gams }} INPUT_IS_GUROBI: ${{ inputs.enable_gurobi }} OS_AUTH_URL: 'https://pouta.csc.fi:5001/v3'