Update ci_build_merge_manual.yml #5
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Manual Full Archive Execution/HTML Gen/Deply | ||
| on: | ||
| workflow_call: | ||
| inputs: | ||
| python-version: | ||
| required: true | ||
| type: string | ||
| secrets: | ||
| CASJOBS_USERID: | ||
| description: 'CASJOBS user ID' | ||
| required: false | ||
| CASJOBS_PW: | ||
| description: 'CASJOBS password' | ||
| required: false | ||
| env: | ||
| CASJOBS_PW: ${{ secrets.CASJOBS_PW }} | ||
| CASJOBS_USERID: ${{ secrets.CASJOBS_USERID }} | ||
| jobs: | ||
| find_notebooks: | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| notebook_list: ${{ steps.find.outputs.notebook_list }} | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v3 | ||
| - name: Find notebook files | ||
| id: find | ||
| run: | | ||
| notebook_files=$(find notebooks -name "*.ipynb" | jq -R -s -c 'split("\n")[:-1]') | ||
| #echo "::set-output name=notebook_list::$notebook_files" | ||
| echo "notebook_list=$notebook_files" >>$GITHUB_OUTPUT | ||
| execute_notebooks: | ||
| needs: find_notebooks | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| executed_notebooks: ${{ steps.execute.outputs.executed_notebooks }} | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| notebook: ${{fromJson(needs.find_notebooks.outputs.notebook_list)}} | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v3 | ||
| - name: Set up Python ${{ inputs.python-version }} | ||
| uses: actions/setup-python@v4 | ||
| with: | ||
| python-version: ${{ inputs.python-version }} | ||
| - name: Install dependencies | ||
| continue-on-error: true | ||
| run: | | ||
| export PYDEVD_DISABLE_FILE_VALIDATION=1 | ||
| export SKLEARN_ALLOW_DEPRECATED_SKLEARN_PACKAGE_INSTALL=True | ||
| python -m pip install --upgrade pip | ||
| pip install pytest | ||
| pip install nbval | ||
| pip install nbconvert | ||
| echo "Path to req's: $(dirname ${{ matrix.notebook }})/requirements.txt" | ||
| ls $(dirname ${{ matrix.notebook }}) | ||
| echo --- | ||
| if [ -f $(dirname "${{ matrix.notebook }}")/pre-installl.sh ]; then | ||
| chmod +x $(dirname "${{ matrix.notebook }}")/pre-install.sh | ||
| ./$(dirname "${{ matrix.notebook }}")/pre-install.sh | ||
| fi | ||
| if [ -f $(dirname "${{ matrix.notebook }}")/pre-requirements.sh ]; then | ||
| chmod +x $(dirname "${{ matrix.notebook }}")/pre-requirements.sh | ||
| ./$(dirname "${{ matrix.notebook }}")/pre-requirements.sh | ||
| fi | ||
| if [ -f pre-requirements.txt ]; then | ||
| pip install -r pre-requirements.txt | ||
| fi | ||
| pip install -r $(dirname ${{ matrix.notebook }})/requirements.txt | ||
| - name: Execute notebook | ||
| id: execute | ||
| run: | | ||
| export PYDEVD_DISABLE_FILE_VALIDATION=1 | ||
| export SKLEARN_ALLOW_DEPRECATED_SKLEARN_PACKAGE_INSTALL=True | ||
| if [ "${{ matrix.notebook }}" != "notebooks/preimaging/preimaging_01_mirage.ipynb" ]; then | ||
| if ! jupyter nbconvert --to notebook --execute --inplace ${{ matrix.notebook }}; then | ||
| python .github/scripts/insert_failure_message.py ${{ matrix.notebook }} | ||
| fi | ||
| fi | ||
| - name: Upload notebook to gh-storage | ||
| run: | | ||
| # Configure Git with the GitHub Actions identity | ||
| git config --global user.name "GitHub Actions" | ||
| git config --global user.email "[email protected]" | ||
| # Create a unique identifier for the temporary branch | ||
| TEMP_BRANCH="temp-$(date +%s)-${{ github.run_id }}" | ||
| # Check out to the temporary branch | ||
| git checkout -b $TEMP_BRANCH | ||
| # Add the notebook | ||
| git add ${{ matrix.notebook }} | ||
| # Commit the changes | ||
| git commit -m "Executed notebook on $TEMP_BRANCH" | ||
| # Retry mechanism | ||
| RETRIES=3 | ||
| DELAY=10 | ||
| until git push origin $TEMP_BRANCH:gh-storage -f; do | ||
| if [ $RETRIES -le 0 ]; then | ||
| echo "Retry limit reached, not trying anymore." | ||
| exit 1 | ||
| fi | ||
| echo "Push failed, retrying in $DELAY seconds..." | ||
| sleep $DELAY | ||
| let RETRIES-=1 | ||
| DELAY=$(( DELAY*2 )) | ||
| git fetch origin | ||
| git reset --hard origin/gh-storage | ||
| done | ||
| # Optionally delete the temporary branch if you want to clean up | ||
| git branch -d $TEMP_BRANCH | ||
| fi | ||
| generate_html: | ||
| runs-on: ubuntu-latest | ||
| needs: execute_notebooks | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v3 | ||
| - name: Set up Python ${{ inputs.python-version }} | ||
| uses: actions/setup-python@v4 | ||
| with: | ||
| python-version: ${{ inputs.python-version }} | ||
| - name: Install dependencies | ||
| run: | | ||
| pip install jupyter-book | ||
| - name: Build HTML | ||
| run: | | ||
| git fetch | ||
| git checkout origin/gh-storage -- notebooks/ | ||
| jupyter-book build . | ||
| # Push the book's HTML to github-pages | ||
| - name: GitHub Pages action | ||
| uses: peaceiris/actions-gh-pages@v3 | ||
| with: | ||
| github_token: ${{ secrets.GITHUB_TOKEN }} | ||
| publish_dir: ./_build/html | ||
