Create data dump #54
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Create data dump | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| schema-version: | |
| required: true | |
| description: Release schema version | |
| type: choice | |
| options: | |
| - v1 | |
| - v2 | |
| new-release: | |
| type: string | |
| description: Name of the directory that the new release is located in | |
| required: true | |
| prev-release: | |
| type: string | |
| description: Name of the existing release zip file to base this data dump from | |
| required: true | |
| jobs: | |
| generate-dump: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Echo message | |
| id: echo_message | |
| run: echo "Github action triggered with inputs new release ${{github.event.inputs.new-release}} and previous release ${{github.event.inputs.prev-release}}" | |
| - name: checkout ror records repo | |
| uses: actions/checkout@v2 | |
| with: | |
| path: ./ror-records | |
| - name: checkout ror data repo | |
| uses: actions/checkout@v2 | |
| with: | |
| repository: ror-community/ror-data-test | |
| token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} | |
| path: ./ror-data | |
| - name: copy previous data dump file | |
| id: copyprevdump | |
| run: | | |
| cp -R ./ror-data/${{github.event.inputs.prev-release}}.zip ./ror-records | |
| - name: checkout ror curation ops repo | |
| uses: actions/checkout@v2 | |
| with: | |
| repository: ror-community/curation_ops | |
| token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} | |
| ref: v2-crosswalk | |
| path: ./curation_ops | |
| - name: Set up Python environment | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: "3.8" | |
| - name: execute script | |
| id: gendumpscript | |
| run: | | |
| cd ./curation_ops/generate_dump/ | |
| if [[ ${{ github.event.inputs.schema-version }} == 'v1' ]]; then | |
| python generate_dump.py -r ${{github.event.inputs.new-release}} -e ${{github.event.inputs.prev-release}} -i '../../ror-records' -o '../../ror-records' -v 1 | |
| fi | |
| if [[ ${{ github.event.inputs.schema-version }} == 'v2' ]]; then | |
| python generate_dump.py -r ${{github.event.inputs.new-release}} -e ${{github.event.inputs.prev-release}} -i '../../ror-records' -o '../../ror-records' -v 2 | |
| fi | |
| - name: cat error file | |
| if: ${{ steps.gendumpscript.outcome != 'success'}} | |
| run: | | |
| echo "ERRORS found:" | |
| cat errors.log | |
| - name: copy new data dump file | |
| id: copynewdump | |
| run: | | |
| yes | cp -rf ./ror-records/${{github.event.inputs.new-release}}*.zip ./ror-data | |
| - name: commit dump file | |
| id: commitdumpfile | |
| if: ${{ steps.copynewdump.outcome == 'success'}} | |
| run: | | |
| cd ./ror-data | |
| git config --local user.email "[email protected]" | |
| git config --local user.name "ror-bot" | |
| git add *.zip | |
| git commit -m "add new data dump file" | |
| git push origin main | |
| - name: commit changed files | |
| if: ${{ steps.commitdumpfile.outcome == 'success'}} && ${{ github.event.inputs.schema-version }} == 'v2' | |
| run: | | |
| cd ./ror-records | |
| git config --local user.email "[email protected]" | |
| git config --local user.name "ror-bot" | |
| git add ${{github.event.inputs.new-release}}/v1/ | |
| git commit -m "add generated v1 files" | |
| git push origin main | |
| - name: Notify Slack | |
| if: always() | |
| uses: edge/simple-slack-notify@master | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.CURATOR_SLACK_WEBHOOK_URL }} | |
| with: | |
| channel: '#ror-curation-releases' | |
| color: 'good' | |
| text: 'PROD Data dump ${{github.event.inputs.new-release}} generation status: ${{ steps.commitdumpfile.outcome }}. Using base version: ${{ github.event.inputs.schema-version }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}' |