refresh-one-notebook.yaml

name: Refresh One Notebook
on:
  workflow_dispatch:
    inputs:
      notebook:
        # --- refresh_notebooks_list.py: refresh list of notebooks currently available [START] ---
        # --- do not edit, auto generated part by `make refresh_notebooks_list` ---
        description: "Notebook file name only in: \n
        - CifarInFhe \n
        - CifarInFheWithSmallerAccumulators \n
        - CifarQuantizationAwareTraining \n
        - ClassifierComparison \n
        - ClientServer \n
        - ConvolutionalNeuralNetwork \n
        - CreditScoring \n
        - DecisionTreeClassifier \n
        - DecisionTreeRegressor \n
        - EncryptedPandas \n
        - ExperimentPrivacyTreePaper \n
        - FromImageNetToCifar \n
        - FullyConnectedNeuralNetwork \n
        - FullyConnectedNeuralNetworkOnMNIST \n
        - GLMComparison \n
        - GPT2FineTuneHybrid \n
        - HealthCarePrediction \n
        - ImportingFromScikitLearn \n
        - KaggleTitanic \n
        - KNearestNeighbors \n
        - LinearRegression \n
        - LinearSVR \n
        - LogisticRegression \n
        - LogisticRegressionTraining \n
        - LoraMLP \n
        - PerrorImpactOnFMNIST \n
        - PoissonRegression \n
        - QGPT2Evaluate \n
        - QuantFrameworkExample \n
        - QuantizationAwareTraining \n
        - RegressorComparison \n
        - SentimentClassification \n
        - SVMClassifier \n
        - WhitePaperExperiments \n
        - XGBClassifier \n
        - XGBRegressor \n
        "
        # --- refresh_notebooks_list.py: refresh list of notebooks currently available [END] ---
        required: true

concurrency:
  group: "${{ github.ref }}-${{ github.event_name }}-${{ github.workflow }}-${{ github.event.inputs.notebook }}"
  cancel-in-progress: false

env:
  ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
  # --- refresh_notebooks_list.py: refresh list of notebook paths currently available [START] ---
  # --- do not edit, auto generated part by `make refresh_notebooks_list` ---
  CifarInFhe: "use_case_examples/cifar/cifar_brevitas_finetuning/CifarInFhe.ipynb" 
  CifarInFheWithSmallerAccumulators: "use_case_examples/cifar/cifar_brevitas_finetuning/CifarInFheWithSmallerAccumulators.ipynb" 
  CifarQuantizationAwareTraining: "use_case_examples/cifar/cifar_brevitas_finetuning/CifarQuantizationAwareTraining.ipynb" 
  ClassifierComparison: "docs/advanced_examples/ClassifierComparison.ipynb" 
  ClientServer: "docs/advanced_examples/ClientServer.ipynb" 
  ConvolutionalNeuralNetwork: "docs/advanced_examples/ConvolutionalNeuralNetwork.ipynb" 
  CreditScoring: "use_case_examples/credit_scoring/CreditScoring.ipynb" 
  DecisionTreeClassifier: "docs/advanced_examples/DecisionTreeClassifier.ipynb" 
  DecisionTreeRegressor: "docs/advanced_examples/DecisionTreeRegressor.ipynb" 
  EncryptedPandas: "docs/advanced_examples/EncryptedPandas.ipynb" 
  ExperimentPrivacyTreePaper: "docs/advanced_examples/ExperimentPrivacyTreePaper.ipynb" 
  FromImageNetToCifar: "use_case_examples/cifar/cifar_brevitas_finetuning/FromImageNetToCifar.ipynb" 
  FullyConnectedNeuralNetwork: "docs/advanced_examples/FullyConnectedNeuralNetwork.ipynb" 
  FullyConnectedNeuralNetworkOnMNIST: "docs/advanced_examples/FullyConnectedNeuralNetworkOnMNIST.ipynb" 
  GLMComparison: "docs/advanced_examples/GLMComparison.ipynb" 
  GPT2FineTuneHybrid: "use_case_examples/lora_finetuning/GPT2FineTuneHybrid.ipynb" 
  HealthCarePrediction: "use_case_examples/disease_prediction/HealthCarePrediction.ipynb" 
  ImportingFromScikitLearn: "docs/advanced_examples/ImportingFromScikitLearn.ipynb" 
  KaggleTitanic: "use_case_examples/titanic/KaggleTitanic.ipynb" 
  KNearestNeighbors: "docs/advanced_examples/KNearestNeighbors.ipynb" 
  LinearRegression: "docs/advanced_examples/LinearRegression.ipynb" 
  LinearSVR: "docs/advanced_examples/LinearSVR.ipynb" 
  LogisticRegression: "docs/advanced_examples/LogisticRegression.ipynb" 
  LogisticRegressionTraining: "docs/advanced_examples/LogisticRegressionTraining.ipynb" 
  LoraMLP: "docs/advanced_examples/LoraMLP.ipynb" 
  PerrorImpactOnFMNIST: "use_case_examples/cifar/cifar_brevitas_finetuning/PerrorImpactOnFMNIST.ipynb" 
  PoissonRegression: "docs/advanced_examples/PoissonRegression.ipynb" 
  QGPT2Evaluate: "use_case_examples/llm/QGPT2Evaluate.ipynb" 
  QuantFrameworkExample: "use_case_examples/llm/QuantFrameworkExample.ipynb" 
  QuantizationAwareTraining: "docs/advanced_examples/QuantizationAwareTraining.ipynb" 
  RegressorComparison: "docs/advanced_examples/RegressorComparison.ipynb" 
  SentimentClassification: "use_case_examples/sentiment_analysis_with_transformer/SentimentClassification.ipynb" 
  SVMClassifier: "docs/advanced_examples/SVMClassifier.ipynb" 
  WhitePaperExperiments: "use_case_examples/white_paper_experiment/WhitePaperExperiments.ipynb" 
  XGBClassifier: "docs/advanced_examples/XGBClassifier.ipynb" 
  XGBRegressor: "docs/advanced_examples/XGBRegressor.ipynb" 
  # --- refresh_notebooks_list.py: refresh list of notebook paths currently available [END] ---

jobs:
  start-runner-linux:
    name: Start EC2 runner
    runs-on: ubuntu-20.04
    outputs:
      label-38: ${{ steps.start-ec2-runner-38.outputs.label }}
      ec2-instance-id-38: ${{ steps.start-ec2-runner-38.outputs.ec2-instance-id || '' }}
    steps:
      - name: Checkout Code
        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ secrets.AWS_REGION }}

      - name: Start EC2 runner python 38
        id: start-ec2-runner-38
        uses: machulav/ec2-github-runner@fcfb31a5760dad1314a64a0e172b78ec6fc8a17e
        with:
          mode: start
          github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }}
          ec2-image-id: ${{ secrets.AWS_EC2_AMI }}
          ec2-instance-type: "m6i.metal"
          subnet-id: ${{ secrets.AWS_EC2_SUBNET_ID }}
          security-group-id: ${{ secrets.AWS_EC2_SECURITY_GROUP_ID }}

  refresh-one-notebook:
    needs: [start-runner-linux]

    runs-on: ${{ needs.start-runner-linux.outputs.label-38 }}
    # Run in a clean container
    container:
      image: ubuntu:20.04
    defaults:
      run:
        shell: bash
    env:
      PIP_INDEX_URL: ${{ secrets.PIP_INDEX_URL }}
      PIP_EXTRA_INDEX_URL: ${{ secrets.PIP_EXTRA_INDEX_URL }}
      NOTEBOOK_NAME: ${{ github.event.inputs.notebook }}
      KAGGLE_USERNAME: ${{ secrets.KAGGLE_USERNAME }}
      KAGGLE_KEY: ${{ secrets.KAGGLE_KEY }}

    steps:
      - name: Add masks
        run: |
          echo "::add-mask::${{ secrets.INTERNAL_PYPI_URL_FOR_MASK }}"
          echo "::add-mask::${{ secrets.INTERNAL_REPO_URL_FOR_MASK }}"

      # Replace default archive.ubuntu.com from docker image with fr mirror
      # original archive showed performance issues and is farther away
      - name: Docker container related setup and git installation
        run: |
          TZ=Europe/Paris
          echo "TZ=${TZ}" >> "$GITHUB_ENV"
          ln -snf /usr/share/zoneinfo/${TZ} /etc/localtime && echo ${TZ} > /etc/timezone
          sed -i 's|^deb http://archive|deb http://fr.archive|g' /etc/apt/sources.list
          apt update && apt install git git-lfs -y

      - name: Checkout Code
        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
        with:
          token: ${{ secrets.BOT_TOKEN }}
          lfs: true

      - name: 'Set up Python 3.8'
        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
        with:
          python-version: '3.8'

      - name: Install dependencies
        run: |
          ./script/make_utils/setup_os_deps.sh
          make setup_env

      - name: Download Sentiment classification data-sets
        if: ${{ env.NOTEBOOK_NAME == 'SentimentClassification' }}
        run: |
          source .venv/bin/activate
          cd ./use_case_examples/encrypted_sentiment_analysis && ./download_data.sh

      - name: Download Titanic data-sets
        if: ${{ env.NOTEBOOK_NAME == 'KaggleTitanic' }}
        run: |
          source .venv/bin/activate
          cd ./use_case_examples/titanic && ./download_data.sh
          
      - name: Refresh ${{ github.event.inputs.notebook }}
        run: |
          make jupyter_execute_one NOTEBOOK="${{ env[env.NOTEBOOK_NAME] }}"
      
      # Pull the latest changes if there are some
      - name: Pull latest changes
        run: |
          git pull -X theirs

      # If the target branch is main or a release branch, a Pull Request is opened for everyone to 
      # review.
      - name: Open PR
        if: ${{ github.ref_name == 'main' || startsWith(github.ref_name , 'release/') }} 
        uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f
        with:
          token: ${{ secrets.BOT_TOKEN }}
          commit-message: "chore: refresh ${{ github.event.inputs.notebook }} notebook"
          branch: "refresh-${{ github.event.inputs.notebook }}-notebook-for-branch-${{ github.ref_name }}"
          base: "${{ github.ref_name }}"
          title: "Refresh ${{ github.event.inputs.notebook }} notebook for branch ${{ github.ref_name }}"
          body: "Automatic PR with notebook refresh of ${{ github.event.inputs.notebook }} \
            for branch ${{ github.ref_name }}."
          add-paths: |
            docs/**/*.ipynb
            use_case_examples/**/*.ipynb
      
      # If the target branch is another branch, the current branch is automatically merged into it
      - name: Push changes into the current branch
        if: ${{ github.ref_name != 'main' && !(startsWith(github.ref_name , 'release/')) }}
        uses: stefanzweifel/git-auto-commit-action@8621497c8c39c72f3e2a999a26b4ca1b5058a842 #v5.0.1
        with:
          commit_message: "chore: refresh ${{ github.event.inputs.notebook }} notebook"
          add_options: '-u'
          file_pattern: |
            docs/**/*.ipynb
            use_case_examples/**/*.ipynb


  stop-runner-linux:
    name: Stop EC2 runner
    needs: [refresh-one-notebook, start-runner-linux]
    runs-on: ubuntu-20.04
    if: ${{ always() && (needs.start-runner-linux.result != 'skipped') }}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ secrets.AWS_REGION }}

      - name: Stop EC2 runner python 38
        uses: machulav/ec2-github-runner@fcfb31a5760dad1314a64a0e172b78ec6fc8a17e
        if: ${{ always() && needs.start-runner-linux.outputs.ec2-instance-id-38 }}
        with:
          github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }}
          label: ${{ needs.start-runner-linux.outputs.label-38 }}
          ec2-instance-id: ${{ needs.start-runner-linux.outputs.ec2-instance-id-38 }}
          mode: stop

  send-report:
    if: ${{ github.ref_name == 'main' || startsWith(github.ref_name , 'release/') }} 
    needs:
      [
        start-runner-linux,
        refresh-one-notebook,
        stop-runner-linux,
      ]

    # If the target branch is main or a release branch, a slack notification is sent
    name: Send Slack notification
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332

      - name: Prepare whole job status
        if: ${{ always() }}
        continue-on-error: true
        env:
          NEEDS_JSON: ${{ toJSON(needs) }}
        run: |
          echo "${NEEDS_JSON}" > /tmp/needs_context.json
          JOB_STATUS=$(python3 ./script/actions_utils/actions_combine_status.py \
          --needs_context_json /tmp/needs_context.json)
          echo "JOB_STATUS=${JOB_STATUS}" >> "$GITHUB_ENV"

      - name: Slack Notification
        continue-on-error: true
        uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
        env:
          SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
          SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
          SLACK_COLOR: ${{ env.JOB_STATUS || 'failure' }}
          SLACK_MESSAGE: "Refreshing ${{ github.event.inputs.notebook }} for ${{ github.ref_name }} 
            finished with status ${{ env.JOB_STATUS || 'failure' }} (${{ env.ACTION_RUN_URL }})\n\
            - start-runner-linux: ${{ needs.start-runner-linux.result || 'Did not run.'}}\n\n\
            - refresh-one-notebook: ${{ needs.refresh-one-notebook.result || 'Did not run.' }}\n\n\
            - stop-runner-linux: ${{ needs.stop-runner-linux.result || 'Did not run.'}}"
          SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}