feat: branching schema bump workflows based on major vs minor schema … #2
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Schema Version Bump Workflow | ||
| on: | ||
| workflow_dispatch | ||
| inputs: | ||
| is-major-schema-bump: | ||
| description: "Set to 'true' to kickoff major schema update; otherwise trigger a minor schema update" | ||
| required: true | ||
| default: 'false' | ||
| permissions: | ||
| id-token: write | ||
| contents: write | ||
| pull-requests: write | ||
| jobs: | ||
| ontology-dry-run: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v3 | ||
| - name: Set up Python 3.8 | ||
| uses: actions/setup-python@v1 | ||
| with: | ||
| python-version: 3.8 | ||
| - name: Python cache | ||
| uses: actions/cache@v1 | ||
| with: | ||
| path: ~/.cache/pip | ||
| key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }} | ||
| restore-keys: | | ||
| ${{ runner.os }}-pip- | ||
| - name: Install dependencies | ||
| run: | | ||
| pip install -r scripts/schema_bump_dry_run_ontologies/requirements.txt | ||
| - name: Configure AWS Credentials | ||
| uses: aws-actions/configure-aws-credentials@v2 | ||
| with: | ||
| aws-region: us-west-2 | ||
| role-to-assume: ${{ secrets.AWS_PROD_ROLE_TO_ASSUME }} | ||
| role-duration-seconds: 2700 | ||
| - name: Pull AWS Secrets | ||
| uses: aws-actions/aws-secretsmanager-get-secrets@v1 | ||
| with: | ||
| secret-ids: | ||
| AUTH0_SECRETS, corpora/backend/prod/auth0-secret | ||
| - name: Check if ontology JSON file has changed | ||
| id: changed-files | ||
| uses: tj-actions/changed-files@v36 | ||
| with: | ||
| files: cellxgene_schema_cli/cellxgene_schema/ontology_files/all_ontology.json.gz | ||
| - name: Run Dry-Run Script | ||
| if: steps.changed-files.outputs.any_changed == 'true' | ||
| run: python3 -m scripts.schema_bump_dry_run_ontologies.ontology_bump_dry_run | ||
| env: | ||
| corpus_env: "prod" | ||
| - name: Store JSON with Replaced By Ontology Term Mappings | ||
| if: steps.changed-files.outputs.any_changed == 'true' | ||
| uses: actions/cache/save@v3 | ||
| with: | ||
| path: "./replaced-by.json" | ||
| key: replaced-by-map | ||
| - name: upload dry run to slack | ||
| if: steps.changed-files.outputs.any_changed == 'true' | ||
| run: | | ||
| curl -F [email protected] -F "initial_comment=Ontology Dry Run Report" -F channels=${{secrets.SLACK_CURATOR_REPORTING_CHANNEL}} -H "Authorization: Bearer ${{secrets.SLACK_CURATOR_REPORTING_APP_AUTH}}" https://slack.com/api/files.upload | ||
| gencode-dry-run: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v3 | ||
| - name: Set up Python | ||
| uses: actions/setup-python@v1 | ||
| with: | ||
| python-version: 3.9 | ||
| - name: Python cache | ||
| uses: actions/cache@v1 | ||
| with: | ||
| path: ~/.cache/pip | ||
| key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }} | ||
| restore-keys: | | ||
| ${{ runner.os }}-pip- | ||
| - name: Install dependencies | ||
| run: | | ||
| pip install -r scripts/schema_bump_dry_run_genes/requirements.txt | ||
| - name: Configure AWS Credentials | ||
| uses: aws-actions/configure-aws-credentials@v2 | ||
| with: | ||
| aws-region: us-west-2 | ||
| role-to-assume: ${{ secrets.AWS_PROD_ROLE_TO_ASSUME }} | ||
| role-duration-seconds: 2700 | ||
| - name: Pull AWS Secrets | ||
| uses: aws-actions/aws-secretsmanager-get-secrets@v1 | ||
| with: | ||
| secret-ids: | ||
| AUTH0_SECRETS, corpora/backend/prod/auth0-secret | ||
| - name: Check if any GENCODE csv files have changed | ||
| id: changed-files | ||
| uses: tj-actions/changed-files@v36 | ||
| with: | ||
| files: cellxgene_schema_cli/cellxgene_schema/ontology_files/*.csv.gz | ||
| - name: Run Dry-Run Script | ||
| if: steps.changed-files.outputs.any_changed == 'true' | ||
| run: python3 -m scripts.schema_bump_dry_run_genes.gene_bump_dry_run | ||
| env: | ||
| corpus_env: "prod" | ||
| - name: upload to slack | ||
| if: steps.changed-files.outputs.any_changed == 'true' | ||
| run: | | ||
| curl -F [email protected] -F "initial_comment=Gene Dry Run Report" -F channels=${{secrets | ||
| .SLACK_CURATOR_REPORTING_CHANNEL}} -H "Authorization: Bearer ${{secrets.SLACK_CURATOR_REPORTING_APP_AUTH}}" https://slack.com/api/files.upload | ||
| publish-to-pypi: | ||
| name: Build and publish Python distributions to PyPI | ||
| runs-on: ubuntu-latest | ||
| if: ${{ github.event.inputs.is-major-schema-bump }} == 'false' | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v3 | ||
| - name: Set up Python | ||
| uses: actions/setup-python@v4 | ||
| with: | ||
| python-version: "3.x" | ||
| - name: setup git | ||
| run: | | ||
| git config user.name github-actions | ||
| git config user.email [email protected] | ||
| - name: Bump and tag release version | ||
| run: | | ||
| pip install bumpversion | ||
| pip install wheel | ||
| bumpversion --config-file .bumpversion.cfg minor --allow-dirty | ||
| bumpversion --config-file .bumpversion.cfg prerel --allow-dirty --tag | ||
| - name: Build dist | ||
| run: >- | ||
| make pydist | ||
| # - name: Publish distribution to Test PyPI | ||
| # uses: pypa/gh-action-pypi-publish@release/v1 | ||
| # with: | ||
| # password: ${{ secrets.TEST_PYPI_API_TOKEN }} | ||
| # repository-url: https://test.pypi.org/legacy/ | ||
| # packages-dir: cellxgene_schema_cli/dist/ | ||
| # - name: Install and Test Package from Test PyPI | ||
| # run: | | ||
| # pip install -r cellxgene_schema_cli/requirements.txt | ||
| # pip install --index-url https://test.pypi.org/simple/ cellxgene-schema | ||
| # cellxgene-schema validate cellxgene_schema_cli/tests/fixtures/h5ads/example_valid.h5ad | ||
| # - name: Publish distribution to PyPI | ||
| # uses: pypa/gh-action-pypi-publish@release/v1 | ||
| # with: | ||
| # password: ${{ secrets.PYPI_API_TOKEN }} | ||
| # packages-dir: cellxgene_schema_cli/dist/ | ||
| - name: Push tags | ||
| run: | | ||
| git push origin --follow-tags | ||
| - name: Trigger rebuild of Data Portal Processing Image | ||
| run: | | ||
| curl -X POST https://api.github.com/repos/chanzuckerberg/single-cell-data-portal/dispatches \ | ||
| -H 'Accept: application/vnd.github.everest-preview+json' \ | ||
| --header 'authorization: Bearer ${{ secrets.CZIBUILDBOT_GITHUB_TOKEN }}' \ | ||
| --data '{"event_type": "rebuild-processing"}' | ||
| update-ontology-mappings: | ||
| runs-on: macos-latest | ||
| steps: | ||
| - name: Configure AWS Credentials | ||
| uses: aws-actions/configure-aws-credentials@v2 | ||
| with: | ||
| aws-region: us-west-2 | ||
| role-to-assume: ${{ secrets.AWS_PROD_ROLE_TO_ASSUME }} | ||
| role-duration-seconds: 1800 | ||
| - name: Checkout | ||
| uses: actions/checkout@v3 | ||
| - name: Set up Python | ||
| uses: actions/setup-python@v1 | ||
| with: | ||
| python-version: 3.11 | ||
| - name: Python cache | ||
| uses: actions/cache@v1 | ||
| with: | ||
| path: ~/.cache/pip | ||
| key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }} | ||
| restore-keys: | | ||
| ${{ runner.os }}-pip- | ||
| - name: install homebrew requirements | ||
| run: brew install graphviz | ||
| - name: install requirements | ||
| run: | | ||
| pip install -r scripts/compute_mappings/requirements.txt | ||
| - name: Generate Ontology Scripts | ||
| run: | | ||
| ./scripts/compute_mappings/compute_tissue_and_cell_type_mappings.py | ||
| ./scripts/compute_mappings/compute_ancestor_mapping.py | ||
| - name: Push Updated Ontology Mappings to S3 | ||
| run: | | ||
| aws s3 sync --exclude="*" --include="*ontology_mapping.json" scripts/compute_mappings s3://cellxgene-schema-ref-files-prod/ontology-mappings/backend | ||
| aws s3 sync --exclude="*" --include="*descendants.json" scripts/compute_mappings s3://cellxgene-schema-ref-files-prod/ontology-mappings/frontend | ||
| aws s3 sync --exclude="*" --include="*.gz" cellxgene_schema_cli/cellxgene_schema/ontology_files s3://cellxgene-schema-ref-files-prod/ontology-mappings/gzips | ||
| - name: Trigger Data Portal to pull latest ontology mappings | ||
| run: | | ||
| curl -X POST https://api.github.com/repos/chanzuckerberg/single-cell-data-portal/dispatches \ | ||
| -H 'Accept: application/vnd.github.everest-preview+json' \ | ||
| --header 'authorization: Bearer ${{ secrets.CZIBUILDBOT_GITHUB_TOKEN }}' \ | ||
| --data '{"event_type": "pull-ontology-mappings"}' | ||
| generate-conversion-script: | ||
| runs-on: ubuntu-latest | ||
| needs: [ontology-dry-run, publish-to-pypi] | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v3 | ||
| with: | ||
| ref: ${{ github.event.pull_request.head.ref }} # checking out the last commit to this branch | ||
| - name: Set up Python | ||
| uses: actions/setup-python@v1 | ||
| with: | ||
| python-version: 3.11 | ||
| - name: Python cache | ||
| uses: actions/cache@v1 | ||
| with: | ||
| path: ~/.cache/pip | ||
| key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }} | ||
| restore-keys: | | ||
| ${{ runner.os }}-pip- | ||
| - name: Install dependencies | ||
| run: | | ||
| pip install -r scripts/migration_assistant/requirements.txt | ||
| - name: Retrieve Replaced By Ontology Term Mapping | ||
| uses: actions/cache/restore@v3 | ||
| with: | ||
| path: ./replaced-by.json | ||
| key: replaced-by-map | ||
| - name: Run Conversion Script Generator | ||
| run: | | ||
| make generate-conversion-script | ||
| - name: Get Version | ||
| id: get_version | ||
| run: | | ||
| echo "old_version=$(make show-current-version)" >> $GITHUB_ENV | ||
| - name: setup git | ||
| run: | | ||
| git config user.name github-actions | ||
| git config user.email [email protected] | ||
| - name: remove diff files | ||
| run: | | ||
| git rm --ignore-unmatch ./cellxgene_schema_cli/cellxgene_schema/ontology_files/*_diff.txt | ||
| - name: Bump Patch Version | ||
| if: ${{ github.event.inputs.is-major-schema-bump }} == 'false' | ||
| run: | | ||
| pip install bumpversion | ||
| pip install wheel | ||
| bumpversion --config-file .bumpversion.cfg patch --allow-dirty | ||
| bumpversion --config-file .bumpversion.cfg prerel --allow-dirty --tag | ||
| echo "new_version=$(make show-current-version)" >> $GITHUB_ENV | ||
| - name: Bump RC to Major Version | ||
| if: ${{ github.event.inputs.is-major-schema-bump }} == 'true' | ||
| run: | | ||
| pip install bumpversion | ||
| pip install wheel | ||
| bumpversion --config-file .bumpversion.cfg prerel --allow-dirty --tag | ||
| echo "new_version=$(make show-current-version)" >> $GITHUB_ENV | ||
| - name: Create Pull Request | ||
| uses: peter-evans/create-pull-request@v5 | ||
| with: | ||
| commit-message: 'AUTO: update migrate.py schema_version ${{ env.old_version }}->${{ env.new_version }}' | ||
| title: 'AUTO: update migrate.py schema_version ${{ env.old_version }}->${{ env.new_version }}' | ||
| body: 'This is an automated PR to update migrate.py from schema_version ${{ env.old_version }}->${{ env.new_version }}.' | ||
| branch: auto/update-convert-py-to-${{ env.new_version }} | ||
| base: main | ||
| reviewers: ${{ vars.MIGRATION_REVIEWERS }} | ||
| add-paths: | | ||
| cellxgene_schema_cli/setup.py | ||
| cellxgene_schema_cli/cellxgene_schema/__init__.py | ||
| cellxgene_schema_cli/cellxgene_schema/migrate.py | ||
| .bumpversion.cfg | ||
