schema-version-bump.yml

name: Schema Version Bump Workflow

on:
  workflow_dispatch:
    inputs:
      is-major-schema-bump:
        description: "Set to 'true' to kickoff major schema update; otherwise trigger a minor schema update"
        required: true
        default: 'false'

permissions:
  id-token: write
  contents: write
  pull-requests: write

jobs:
  ontology-dry-run:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Set up Python 3.8
        uses: actions/setup-python@v1
        with:
          python-version: 3.8
      - name: Python cache
        uses: actions/cache@v1
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-
      - name: Install dependencies
        run: |
          pip install -r scripts/schema_bump_dry_run_ontologies/requirements.txt
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v2
        with:
          aws-region: us-west-2
          role-to-assume: ${{ secrets.AWS_PROD_ROLE_TO_ASSUME }}
          role-duration-seconds: 2700
      - name: Pull AWS Secrets
        uses: aws-actions/aws-secretsmanager-get-secrets@v1
        with:
          secret-ids:
            AUTH0_SECRETS, corpora/backend/prod/auth0-secret
      - name: Run Dry-Run Script
        run: python3 -m scripts.schema_bump_dry_run_ontologies.ontology_bump_dry_run
        env:
          corpus_env: "prod"
      - name: Store JSON with Replaced By Ontology Term Mappings
        uses: actions/cache/save@v3
        with:
          path: "./replaced-by.json"
          key: replaced-by-map
      - name: upload dry run to slack
        run: |
          curl -F file=@ontologies-curator-report.txt -F "initial_comment=Ontology Dry Run Report" -F channels=${{secrets.SLACK_CURATOR_REPORTING_CHANNEL}} -H "Authorization: Bearer ${{secrets.SLACK_CURATOR_REPORTING_APP_AUTH}}" https://slack.com/api/files.upload

  gencode-dry-run:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v1
        with:
          python-version: 3.9
      - name: Python cache
        uses: actions/cache@v1
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-
      - name: Install dependencies
        run: |
          pip install -r scripts/schema_bump_dry_run_genes/requirements.txt
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v2
        with:
          aws-region: us-west-2
          role-to-assume: ${{ secrets.AWS_PROD_ROLE_TO_ASSUME }}
          role-duration-seconds: 2700
      - name: Pull AWS Secrets
        uses: aws-actions/aws-secretsmanager-get-secrets@v1
        with:
          secret-ids:
            AUTH0_SECRETS, corpora/backend/prod/auth0-secret
      - name: Check if any GENCODE csv files have changed
        id: changed-files
        uses: tj-actions/changed-files@v36
        with:
          files: cellxgene_schema_cli/cellxgene_schema/ontology_files/*.csv.gz
      - name: Run Dry-Run Script
        if: steps.changed-files.outputs.any_changed == 'true'
        run: python3 -m scripts.schema_bump_dry_run_genes.gene_bump_dry_run
        env:
          corpus_env: "prod"
      - name: upload to slack
        if: steps.changed-files.outputs.any_changed == 'true'
        run: |
          curl -F file=@genes-curator-report.txt -F "initial_comment=Gene Dry Run Report" -F channels=${{secrets
          .SLACK_CURATOR_REPORTING_CHANNEL}} -H "Authorization: Bearer ${{secrets.SLACK_CURATOR_REPORTING_APP_AUTH}}" https://slack.com/api/files.upload

  publish-to-pypi:
    name: Build and publish Python distributions to PyPI
    runs-on: ubuntu-latest
    if: "${{ github.event.inputs.is-major-schema-bump == 'false' }}"
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.x"
      - name: setup git
        run: |
          git config user.name github-actions
          git config user.email github-actions@github.com
#      - name: Bump and tag release version
#        run: |
#          pip install bumpversion
#          pip install wheel
#          bumpversion --config-file .bumpversion.cfg minor --allow-dirty
#          bumpversion --config-file .bumpversion.cfg prerel --allow-dirty --tag
#      - name: Build dist
#        run: >-
#          make pydist
      # - name: Publish distribution to Test PyPI
      #   uses: pypa/gh-action-pypi-publish@release/v1
      #   with:
      #     password: ${{ secrets.TEST_PYPI_API_TOKEN }}
      #     repository-url: https://test.pypi.org/legacy/
      #     packages-dir: cellxgene_schema_cli/dist/
      # - name: Install and Test Package from Test PyPI
      #   run: |
      #     pip install -r cellxgene_schema_cli/requirements.txt
      #     pip install --index-url https://test.pypi.org/simple/ cellxgene-schema
      #     cellxgene-schema validate cellxgene_schema_cli/tests/fixtures/h5ads/example_valid.h5ad
      # - name: Publish distribution to PyPI
      #   uses: pypa/gh-action-pypi-publish@release/v1
      #   with:
      #     password: ${{ secrets.PYPI_API_TOKEN }}
      #     packages-dir: cellxgene_schema_cli/dist/
#      - name: Push tags
#        run: |
#          git push origin --follow-tags
#      - name: Trigger rebuild of Data Portal Processing Image
#        run: |
#          curl -X POST https://api.github.com/repos/chanzuckerberg/single-cell-data-portal/dispatches \
#          -H 'Accept: application/vnd.github.everest-preview+json' \
#          --header 'authorization: Bearer ${{ secrets.CZIBUILDBOT_GITHUB_TOKEN }}' \
#          --data '{"event_type": "rebuild-processing"}'

  update-ontology-mappings:
    runs-on: macos-latest
    steps:
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v2
        with:
          aws-region: us-west-2
          role-to-assume: ${{ secrets.AWS_PROD_ROLE_TO_ASSUME }}
          role-duration-seconds: 1800
      - name: Checkout
        uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v1
        with:
          python-version: 3.11
      - name: Python cache
        uses: actions/cache@v1
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-
      - name: install homebrew requirements
        run: brew install graphviz
      - name: install requirements
        run: |
          pip install -r scripts/compute_mappings/requirements.txt
      - name: Generate Ontology Scripts
        run: |
          ./scripts/compute_mappings/compute_tissue_and_cell_type_mappings.py
          ./scripts/compute_mappings/compute_ancestor_mapping.py
      - name: Push Updated Ontology Mappings to S3
        run: |
          aws s3 sync --exclude="*" --include="*ontology_mapping.json" scripts/compute_mappings s3://cellxgene-schema-ref-files-prod/ontology-mappings/backend
          aws s3 sync --exclude="*" --include="*descendants.json" scripts/compute_mappings s3://cellxgene-schema-ref-files-prod/ontology-mappings/frontend
          aws s3 sync --exclude="*" --include="*.gz" cellxgene_schema_cli/cellxgene_schema/ontology_files s3://cellxgene-schema-ref-files-prod/ontology-mappings/gzips
      - name: Trigger Data Portal to pull latest ontology mappings
        run: |
          curl -X POST https://api.github.com/repos/chanzuckerberg/single-cell-data-portal/dispatches \
          -H 'Accept: application/vnd.github.everest-preview+json' \
          --header 'authorization: Bearer ${{ secrets.CZIBUILDBOT_GITHUB_TOKEN }}' \
          --data '{"event_type": "pull-ontology-mappings"}'

  generate-conversion-script:
    runs-on: ubuntu-latest
    needs: [ontology-dry-run, publish-to-pypi]
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          ref: ${{ github.event.pull_request.head.ref }} # checking out the last commit to this branch
      - name: Set up Python
        uses: actions/setup-python@v1
        with:
          python-version: 3.11
      - name: Python cache
        uses: actions/cache@v1
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-
      - name: Install dependencies
        run: |
          pip install -r scripts/migration_assistant/requirements.txt
      - name: Retrieve Replaced By Ontology Term Mapping
        uses: actions/cache/restore@v3
        with:
          path: ./replaced-by.json
          key: replaced-by-map
      - name: Run Conversion Script Generator
        run: |
          make generate-conversion-script
      - name: Get Version
        id: get_version
        run: |
          echo "old_version=$(make show-current-version)" >> $GITHUB_ENV
      - name: setup git
        run: |
          git config user.name github-actions
          git config user.email github-actions@github.com
      - name: remove diff files
        run: |
          git rm --ignore-unmatch ./cellxgene_schema_cli/cellxgene_schema/ontology_files/*_diff.txt
      - name: Bump Patch Version
        if: ${{ github.event.inputs.is-major-schema-bump }} == 'false'
        run: |
          pip install bumpversion
          pip install wheel
          bumpversion --config-file .bumpversion.cfg patch --allow-dirty
          bumpversion --config-file .bumpversion.cfg prerel --allow-dirty --tag
          echo "new_version=$(make show-current-version)" >> $GITHUB_ENV
      - name: Bump RC to Major Version
        if: ${{ github.event.inputs.is-major-schema-bump }} == 'true'
        run: |
          pip install bumpversion
          pip install wheel
          bumpversion --config-file .bumpversion.cfg prerel --allow-dirty --tag
          echo "new_version=$(make show-current-version)" >> $GITHUB_ENV
      - name: Create Pull Request
        uses: peter-evans/create-pull-request@v5
        with:
          commit-message: 'AUTO: update migrate.py schema_version ${{ env.old_version }}->${{ env.new_version }}'
          title: 'AUTO: update migrate.py schema_version ${{ env.old_version }}->${{ env.new_version }}'
          body: 'This is an automated PR to update migrate.py from schema_version ${{ env.old_version }}->${{ env.new_version }}.'
          branch: auto/update-convert-py-to-${{ env.new_version }}
          base: main
          reviewers: ${{ vars.MIGRATION_REVIEWERS }}
          add-paths: |
            cellxgene_schema_cli/setup.py
            cellxgene_schema_cli/cellxgene_schema/__init__.py
            cellxgene_schema_cli/cellxgene_schema/migrate.py
            .bumpversion.cfg