From 0e9e5643a031a6f8fb8187e67e5c658d7ff29757 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Vall=C3=A9e?= Date: Mon, 31 Jul 2023 10:16:56 +0200 Subject: [PATCH] #361 enhance published release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Frédéric Vallée --- .github/workflows/ci.yml | 26 ++-- .github/workflows/nightly-release.yml | 10 +- .github/workflows/nightly.yml | 2 +- .github/workflows/release.yml | 172 +++++++++++++++++++++----- .github/workflows/tools.sh | 72 ----------- 5 files changed, 156 insertions(+), 126 deletions(-) delete mode 100644 .github/workflows/tools.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a17e2a4..f48cba2d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,21 +18,15 @@ jobs: DYNAWO_RESULTS_SHOW: "false" DYNAWO_FORCE_CXX11_ABI: "true" DYNAWO_NB_PROCESSORS_USED: 2 - GH_API_DYNAWO: ${{ github.api_url }}/repos/${{ github.repository_owner }}/dynawo - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - CURL_JSON: 'curl --silent -H "authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" --header "Content-Type: application/json"' steps: - name: Checkout sources - uses: actions/checkout@v1 + uses: actions/checkout@v3 - name: Fetch Dynawo - run: | - source .github/workflows/tools.sh - download_asset $GH_API_DYNAWO nightly Dynawo_headers_v${DYNAWO_VERSION}.zip $GITHUB_TOKEN + run: curl -LOs https://github.com/${{ github.repository_owner }}/dynawo/releases/download/nightly/Dynawo_headers_v$DYNAWO_VERSION.zip - name: Unzip Dynawo - run: | - unzip "Dynawo_headers_v${DYNAWO_VERSION}.zip" + run: unzip -qq "Dynawo_headers_v${DYNAWO_VERSION}.zip" - name: Dynawo version run: ./dynawo/dynawo.sh version @@ -55,9 +49,6 @@ jobs: DYNAWO_FORCE_CXX11_ABI: "true" DYNAWO_RESULTS_SHOW: "false" DYNAWO_NB_PROCESSORS_USED: 2 - GH_API_DYNAWO: ${{ github.api_url }}/repos/${{ github.repository_owner }}/dynawo - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - CURL_JSON: 'curl --silent -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" --header "Content-Type: application/json"' steps: - name: Install Sonar wrapper working-directory: ${{ runner.workspace }} @@ -79,16 +70,13 @@ jobs: dnf install -y java-11-openjdk - name: Checkout sources - uses: actions/checkout@v1 + uses: actions/checkout@v3 - name: Fetch Dynawo - run: | - source .github/workflows/tools.sh - download_asset $GH_API_DYNAWO nightly Dynawo_omc_v${DYNAWO_VERSION}.zip $GITHUB_TOKEN + run: curl -LOs https://github.com/${{ github.repository_owner }}/dynawo/releases/download/nightly/Dynawo_omc_v$DYNAWO_VERSION.zip - name: Unzip Dynawo - run: | - unzip "Dynawo_omc_v${DYNAWO_VERSION}.zip" + run: unzip -qq "Dynawo_omc_v${DYNAWO_VERSION}.zip" - name: Build env: @@ -134,7 +122,7 @@ jobs: - name: Fetch Dynawo shell: bash - run: curl -LOs https://github.com/dynawo/dynawo/releases/download/nightly/Dynawo_Windows_v$DYNAWO_VERSION.zip + run: curl -LOs https://github.com/${{ github.repository_owner }}/dynawo/releases/download/nightly/Dynawo_Windows_v$DYNAWO_VERSION.zip - name: Unzip Dynawo shell: cmd diff --git a/.github/workflows/nightly-release.yml b/.github/workflows/nightly-release.yml index 5abd4e3c..f7559bde 100644 --- a/.github/workflows/nightly-release.yml +++ b/.github/workflows/nightly-release.yml @@ -41,7 +41,7 @@ jobs: - name: Fetch release script uses: actions/checkout@v3 with: - repository: dynawo/dynawo + repository: ${{ github.repository_owner }}/dynawo sparse-checkout: .github/release.js sparse-checkout-cone-mode: false @@ -79,7 +79,7 @@ jobs: uses: actions/checkout@v3 - name: Fetch Dynawo's nightly release - run: curl -LOs https://github.com/dynawo/dynawo/releases/download/nightly/Dynawo_headers_v$DYNAWO_VERSION.zip + run: curl -LOs https://github.com/${{ github.repository_owner }}/dynawo/releases/download/nightly/Dynawo_headers_v$DYNAWO_VERSION.zip - name: Unzip Dynawo run: unzip -qq "Dynawo_headers_v${DYNAWO_VERSION}.zip" @@ -109,7 +109,7 @@ jobs: - name: Fetch release script uses: actions/checkout@v3 with: - repository: dynawo/dynawo + repository: ${{ github.repository_owner }}/dynawo sparse-checkout: .github/release.js sparse-checkout-cone-mode: false path: dynawo @@ -165,7 +165,7 @@ jobs: - name: Fetch Dynawo's nightly release shell: bash - run: curl -LOs https://github.com/dynawo/dynawo/releases/download/nightly/Dynawo_Windows_v$DYNAWO_VERSION.zip + run: curl -LOs https://github.com/${{ github.repository_owner }}/dynawo/releases/download/nightly/Dynawo_Windows_v$DYNAWO_VERSION.zip - name: Unzip Dynawo shell: cmd @@ -223,7 +223,7 @@ jobs: - name: Fetch release script uses: actions/checkout@v3 with: - repository: dynawo/dynawo + repository: ${{ github.repository_owner }}/dynawo sparse-checkout: .github/release.js sparse-checkout-cone-mode: false path: dynawo diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index fe622296..f9ccf133 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -32,7 +32,7 @@ jobs: - name: Unzip Dynawo run: | - unzip "Dynawo_omc_v${DYNAWO_VERSION}.zip" + unzip -qq "Dynawo_omc_v${DYNAWO_VERSION}.zip" - name: Dynawo version run: ./dynawo/dynawo.sh version diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 17c47005..9402a41f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,10 +3,32 @@ name: Release on: release: types: [published] + workflow_dispatch: + inputs: + release-id: + description: ID of the published release + type: string + required: true + perform-linux-cxx11: + description: Create assets for Linux cxx11 + type: boolean + default: true + perform-linux-centos7: + description: Create assets for Linux centos7 + type: boolean + default: true + perform-windows: + description: Create assets for Windows + type: boolean + default: true + +env: + DYNAWO_VERSION: 1.6.0 jobs: - distribution: - name: Distribution Linux + distribution-cxx11: + if: github.event_name != 'workflow_dispatch' || inputs.perform-linux-cxx11 + name: Distribution Linux cxx11 runs-on: ubuntu-latest container: dynawo/dynawo-distribution-cxx11:latest env: @@ -14,23 +36,16 @@ jobs: DYNAWO_BUILD_TYPE: Release DYNAWO_NB_PROCESSORS_USED: 2 DYNAWO_FORCE_CXX11_ABI: "true" - DYNAWO_VERSION: 1.6.0 - GH_API_DYNAWO: ${{ github.api_url }}/repos/${{ github.repository_owner }}/dynawo - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - CURL_JSON: 'curl --silent -H "authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" --header "Content-Type: application/json"' steps: - name: Checkout sources - uses: actions/checkout@v1 + uses: actions/checkout@v3 - name: Fetch Dynawo - run: | - source .github/workflows/tools.sh - download_asset $GH_API_DYNAWO v${DYNAWO_VERSION} Dynawo_Linux_v${DYNAWO_VERSION}.zip $GITHUB_TOKEN + run: curl -LOs https://github.com/${{ github.repository_owner }}/dynawo/releases/download/v$DYNAWO_VERSION/Dynawo_Linux_v$DYNAWO_VERSION.zip - name: Unzip Dynawo - run: | - unzip "Dynawo_Linux_v${DYNAWO_VERSION}.zip" + run: unzip -qq "Dynawo_Linux_v${DYNAWO_VERSION}.zip" - name: Build run: | @@ -39,11 +54,31 @@ jobs: util/envDynawoAlgorithms.sh build util/envDynawoAlgorithms.sh distrib util/envDynawoAlgorithms.sh distrib-headers - version=$(util/envDynawoAlgorithms.sh version | cut -f1 -d' ') - curl -H "authorization: Bearer $GITHUB_TOKEN" -H "Content-Type: application/zip" -X POST $(curl -s -H "authorization: Bearer $GITHUB_TOKEN" --request GET https://api.github.com/repos/dynawo/dynawo-algorithms/releases/latest | grep upload_url | cut -d '"' -f 4 | grep -o ".*assets")?name=DynawoAlgorithms_Linux_v${version}.zip --data-binary @distributions/DynawoAlgorithms_v${version}.zip - curl -H "authorization: Bearer $GITHUB_TOKEN" -H "Content-Type: application/zip" -X POST $(curl -s -H "authorization: Bearer $GITHUB_TOKEN" --request GET https://api.github.com/repos/dynawo/dynawo-algorithms/releases/latest | grep upload_url | cut -d '"' -f 4 | grep -o ".*assets")?name=DynawoAlgorithms_Linux_headers_v${version}.zip --data-binary @distributions/DynawoAlgorithms_headers_v${version}.zip + for file in distributions/DynawoAlgorithms_v*.zip ; do mv $file ${file//_/_Linux_} ; done + for file in distributions/DynawoAlgorithms_headers_v*.zip ; do mv $file ${file//_headers_/_Linux_headers_} ; done + + - name: Fetch release script + uses: actions/checkout@v3 + with: + repository: ${{ github.repository_owner }}/dynawo + sparse-checkout: .github/release.js + sparse-checkout-cone-mode: false + path: dynawo + + - name: Upload assets + uses: actions/github-script@v6 + env: + release_id: ${{ github.event_name != 'workflow_dispatch' && github.event.release.id || inputs.release-id }} + files: | + distributions/DynawoAlgorithms_*.zip + with: + retries: 3 + script: | + const release = require('./dynawo/.github/release.js') + await release({github, context, core, glob}) distribution-centos7: + if: github.event_name != 'workflow_dispatch' || inputs.perform-linux-centos7 name: Distribution Linux centos7 runs-on: ubuntu-latest container: dynawo/dynawo-distribution-centos7:latest @@ -51,31 +86,110 @@ jobs: DYNAWO_RESULTS_SHOW: "false" DYNAWO_BUILD_TYPE: Release DYNAWO_NB_PROCESSORS_USED: 2 - DYNAWO_VERSION: 1.6.0 - GH_API_DYNAWO: ${{ github.api_url }}/repos/${{ github.repository_owner }}/dynawo - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - CURL_JSON: 'curl --silent -H "authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" --header "Content-Type: application/json"' steps: - name: Checkout sources - uses: actions/checkout@v1 + uses: actions/checkout@v3 - name: Fetch Dynawo - run: | - source .github/workflows/tools.sh - download_asset $GH_API_DYNAWO v${DYNAWO_VERSION} Dynawo_Linux_centos7_v${DYNAWO_VERSION}.zip $GITHUB_TOKEN + run: curl -LOs https://github.com/${{ github.repository_owner }}/dynawo/releases/download/v$DYNAWO_VERSION/Dynawo_Linux_centos7_v$DYNAWO_VERSION.zip - name: Unzip Dynawo - run: | - unzip "Dynawo_Linux_centos7_v${DYNAWO_VERSION}.zip" + run: unzip -qq "Dynawo_Linux_centos7_v${DYNAWO_VERSION}.zip" - name: Build - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | export DYNAWO_HOME=$(pwd)/dynawo export DYNAWO_ALGORITHMS_HOME=$(pwd) util/envDynawoAlgorithms.sh build util/envDynawoAlgorithms.sh distrib-headers - version=$(util/envDynawoAlgorithms.sh version | cut -f1 -d' ') - curl -H "authorization: Bearer $GITHUB_TOKEN" -H "Content-Type: application/zip" -X POST $(curl -s -H "authorization: Bearer $GITHUB_TOKEN" --request GET https://api.github.com/repos/dynawo/dynawo-algorithms/releases/latest | grep upload_url | cut -d '"' -f 4 | grep -o ".*assets")?name=DynawoAlgorithms_Linux_headers_centos7_v${version}.zip --data-binary @distributions/DynawoAlgorithms_headers_v${version}.zip + for file in distributions/DynawoAlgorithms_headers_v*.zip ; do mv $file ${file//_headers_/_Linux_headers_centos7_} ; done + + - name: Fetch release script + uses: actions/checkout@v3 + with: + repository: ${{ github.repository_owner }}/dynawo + sparse-checkout: .github/release.js + sparse-checkout-cone-mode: false + path: dynawo + + - name: Upload assets + uses: actions/github-script@v6 + env: + release_id: ${{ github.event_name != 'workflow_dispatch' && github.event.release.id || inputs.release-id }} + files: | + distributions/DynawoAlgorithms_*.zip + with: + retries: 3 + script: | + const release = require('./dynawo/.github/release.js') + await release({github, context, core, glob}) + + distribution-windows: + if: github.event_name != 'workflow_dispatch' || inputs.perform-windows + name: Distribution Windows + runs-on: windows-2019 + steps: + - name: No LF to CRLF conversion + shell: cmd + run: git config --global core.autocrlf input + + - name: Checkout sources + uses: actions/checkout@v3 + + - name: Download MS-MPI + shell: bash + run: | + curl -Os https://download.microsoft.com/download/a/5/2/a5207ca5-1203-491a-8fb8-906fd68ae623/msmpisdk.msi + curl -Os https://download.microsoft.com/download/a/5/2/a5207ca5-1203-491a-8fb8-906fd68ae623/msmpisetup.exe + + - name: Install MS-MPI + shell: cmd + run: | + msmpisdk.msi /passive + msmpisetup.exe /unattend + + - name: Fetch Dynawo + shell: bash + run: curl -LOs https://github.com/${{ github.repository_owner }}/dynawo/releases/download/v$DYNAWO_VERSION/Dynawo_Windows_v$DYNAWO_VERSION.zip + + - name: Unzip Dynawo + shell: cmd + run: 7z x Dynawo_Windows_v%DYNAWO_VERSION%.zip -o..\deploy + + # use a minimal PATH to avoid too long command lines and conflict with Strawberry pkg-config ! + - name: Build and install + shell: cmd + env: + MSMPI_INC: C:\Program Files (x86)\Microsoft SDKs\MPI\Include\ + MSMPI_LIB32: C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\ + MSMPI_LIB64: C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x64\ + PATH: C:\Windows\system32;C:\Program Files\Git\cmd;C:\hostedtoolcache\windows\Python\3.7.9\x64 + run: | + call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat" + call util\windows\dynawo-algorithms verbose build || exit /b 1 + call util\windows\dynawo-algorithms verbose distrib-headers distrib || exit /b 1 + + - name: Rename distribution + shell: bash + run: for file in distrib/DynawoAlgorithms_*.zip ; do mv $file ${file//headers_Release/Windows} ; done + + - name: Fetch release script + uses: actions/checkout@v3 + with: + repository: ${{ github.repository_owner }}/dynawo + sparse-checkout: .github/release.js + sparse-checkout-cone-mode: false + path: dynawo + + - name: Upload assets + uses: actions/github-script@v6 + env: + release_id: ${{ github.event_name != 'workflow_dispatch' && github.event.release.id || inputs.release-id }} + files: | + distrib/DynawoAlgorithms_*.zip + with: + retries: 3 + script: | + const release = require('./dynawo/.github/release.js') + await release({github, context, core, glob}) diff --git a/.github/workflows/tools.sh b/.github/workflows/tools.sh deleted file mode 100644 index 4e9f95bd..00000000 --- a/.github/workflows/tools.sh +++ /dev/null @@ -1,72 +0,0 @@ -wait_for_http_code() -{ - local WAIT_URL=$1 - local WAIT_CODE=$2 - local TOKEN=$3 - echo "wait_for_http_code($WAIT_URL, $WAIT_CODE)" - - local CODE="" - local MAX_RETRIES=20 - local RETRIES=0 - while [[ (-z "$CODE" || "$CODE" -ne "$WAIT_CODE") && ("$RETRIES" -lt "$MAX_RETRIES") ]] - do - sleep 1 - CODE=$(curl --silent --header "authorization: Bearer $TOKEN" --head "$WAIT_URL" | grep ^HTTP | tr -s ' ' | cut -f 2 -d' ') - RETRIES=`expr $RETRIES + 1` - echo "DEBUG. Waiting for $WAIT_URL = $WAIT_CODE. Retries $RETRIES, code $CODE" - done - echo "DEBUG. Waiting finished. Retries $RETRIES, code $CODE" -} - -upload_zip_file() -{ - local UPLOAD_URL=$1 - local FILE=$2 - local TOKEN=$3 - echo "upload_zip_file($UPLOAD_URL, $FILE)" - - # Limit rate is used to ensure uploads finish - # When not used, systematic errors for TCP connection reset are received - curl \ - --retry 20 --retry-delay 0 --retry-max-time 40 --max-time 180 --limit-rate 5M \ - --request POST \ - --url $UPLOAD_URL?name=$FILE \ - --header "authorization: Bearer $TOKEN" \ - --header "Content-Type: application/zip" \ - --data-binary @$FILE -} - -download_asset() { - local GH_API_REPO=$1 - local RELEASE_TAG=$2 - local ASSET_NAME=$3 - local TOKEN=$4 - local ASSET_URL=$(eval $CURL_JSON $GH_API_REPO/releases/tags/$RELEASE_TAG | python -c \ - "import sys, json; assets = [a for a in json.load(sys.stdin)['assets'] if a['name'] == '$ASSET_NAME']; print(assets[0]['browser_download_url']) if assets else ''" \ - ) - echo "INFO. asset url:" - echo "$ASSET_URL" - echo "" - if [ ! -z $ASSET_URL ]; then - download_file $ASSET_URL $ASSET_NAME $TOKEN - else - echo "Asset $ASSET_NAME not found in $RELEASE_TAG release in repository $REPO" - exit 1 - fi -} - -download_file() -{ - local DOWNLOAD_URL=$1 - local FILENAME=$2 - local TOKEN=$3 - curl \ - --retry 20 --retry-delay 0 --retry-max-time 40 --max-time 180 --limit-rate 5M \ - --remote-header-name \ - --location \ - --header "authorization: Bearer $TOKEN" \ - --header "Accept: application/octet-stream" \ - --url "$DOWNLOAD_URL" \ - --create-dirs \ - --output "$FILENAME" -}