Skip to content

Commit

Permalink
Merge pull request #150 from ror-community/staging
Browse files Browse the repository at this point in the history
Merge v1.45 to production
  • Loading branch information
adambuttrick authored Apr 11, 2024
2 parents 7cd8fc4 + 8492bac commit 3bc3bb7
Show file tree
Hide file tree
Showing 225 changed files with 24,265 additions and 54 deletions.
34 changes: 28 additions & 6 deletions .github/workflows/generate_dump.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,13 @@ name: Create data dump
on:
workflow_dispatch:
inputs:
schema-version:
required: true
description: Release schema version
type: choice
options:
- v1
- v2
new-release:
type: string
description: Name of the directory that the new release is located in
Expand All @@ -25,7 +32,7 @@ jobs:
- name: checkout ror data repo
uses: actions/checkout@v2
with:
repository: ror-community/ror-data
repository: ror-community/ror-data-test
token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
path: ./ror-data
- name: copy previous data dump file
Expand All @@ -37,6 +44,7 @@ jobs:
with:
repository: ror-community/curation_ops
token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
ref: v2-crosswalk
path: ./curation_ops
- name: Set up Python environment
uses: actions/setup-python@v2
Expand All @@ -46,7 +54,12 @@ jobs:
id: gendumpscript
run: |
cd ./curation_ops/generate_dump/
python generate_dump.py -r ${{github.event.inputs.new-release}} -e ${{github.event.inputs.prev-release}} -i '../../ror-records' -o '../../ror-records'
if [[ ${{ github.event.inputs.schema-version }} == 'v1' ]]; then
python generate_dump.py -r ${{github.event.inputs.new-release}} -e ${{github.event.inputs.prev-release}} -i '../../ror-records' -o '../../ror-records' -v 1
fi
if [[ ${{ github.event.inputs.schema-version }} == 'v2' ]]; then
python generate_dump.py -r ${{github.event.inputs.new-release}} -e ${{github.event.inputs.prev-release}} -i '../../ror-records' -o '../../ror-records' -v 2
fi
- name: cat error file
if: ${{ steps.gendumpscript.outcome != 'success'}}
run: |
Expand All @@ -61,11 +74,20 @@ jobs:
if: ${{ steps.copynewdump.outcome == 'success'}}
run: |
cd ./ror-data
git config --local user.email "[email protected]"
git config --local user.name "GitHub Action"
git config --local user.email "[email protected]"
git config --local user.name "ror-bot"
git add *.zip
git commit -m "add new data dump file"
git push origin main
git push origin
- name: commit changed files
if: ${{ steps.commitdumpfile.outcome == 'success'}} && ${{ github.event.inputs.schema-version }} == 'v2'
run: |
cd ./ror-records
git config --local user.email "[email protected]"
git config --local user.name "ror-bot"
git add ${{github.event.inputs.new-release}}/v1/
git commit -m "add generated v1 files"
git push origin ${{ github.event.ref }}
- name: Notify Slack
if: always()
uses: edge/simple-slack-notify@master
Expand All @@ -74,4 +96,4 @@ jobs:
with:
channel: '#ror-curation-releases'
color: 'good'
text: 'PROD Data dump ${{github.event.inputs.new-release}} generation status: ${{ steps.commitdumpfile.outcome }} Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
text: 'PROD Data dump ${{github.event.inputs.new-release}} generation status: ${{ steps.commitdumpfile.outcome }}. Using base version: ${{ github.event.inputs.schema-version }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
12 changes: 6 additions & 6 deletions .github/workflows/main_release.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Deploy to Prod
name: Deploy to Prod v2 only
on:
release:
types: [published]
Expand Down Expand Up @@ -50,8 +50,8 @@ jobs:
run: |
cd ./${{ github.event.release.tag_name }}
zip files.zip *.json
aws s3 cp files.zip s3://${{ secrets.ROR_DATA_S3_BUCKET_PROD }}/${{ github.event.release.tag_name }}/files.zip
aws s3 ls s3://${{ secrets.ROR_DATA_S3_BUCKET_PROD }}/${{ github.event.release.tag_name }}/files.zip
aws s3 cp files.zip s3://${{ secrets.ROR_DATA_S3_BUCKET_PROD }}/${{ github.event.release.tag_name }}-v2/files.zip
aws s3 ls s3://${{ secrets.ROR_DATA_S3_BUCKET_PROD }}/${{ github.event.release.tag_name }}-v2/files.zip
- name: Notify Slack
if: always()
uses: edge/simple-slack-notify@master
Expand All @@ -60,14 +60,14 @@ jobs:
with:
channel: '#ror-curation-releases'
color: 'good'
text: 'Copy status to PROD S3 Bucket: ${{ steps.copyfiles.outcome }}. From directory: ${{ github.event.release.tag_name }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
text: 'Copy status to PROD S3 Bucket: ${{ steps.copyfiles.outcome }}. Using version 2. From directory: ${{ github.event.release.tag_name }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
- name: Index data
id: indexdata
run: |
cd .github/workflows
python -m pip install --upgrade pip
pip install requests==2.23.0
python index_files.py -u ${{ secrets.INDEX_PROD_API_URL }} -d ${{ github.event.release.tag_name }} -he ${{ secrets.INDEX_PROD_API_HEADERS }}
python index_files.py -u ${{ secrets.INDEX_PROD_API_URL_V2 }} -d ${{ github.event.release.tag_name }}-v2 -he ${{ secrets.INDEX_PROD_API_HEADERS }}
- name: Notify Slack
if: always()
uses: edge/simple-slack-notify@master
Expand All @@ -76,4 +76,4 @@ jobs:
with:
channel: '#ror-curation-releases'
color: 'good'
text: 'PROD index status: ${{ steps.indexdata.outcome }}. From directory: ${{ github.event.release.tag_name }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
text: 'PROD index status: ${{ steps.indexdata.outcome }}. Using version 2. From directory: ${{ github.event.release.tag_name }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
2 changes: 1 addition & 1 deletion .github/workflows/prod_index_dump.yml
Original file line number Diff line number Diff line change
Expand Up @@ -74,5 +74,5 @@ jobs:
with:
channel: '#ror-curation-releases'
color: 'good'
text: 'PROD full index from dump status: ${{ steps.indexdatadump.outcome }}. From file: ${{ github.event.inputs.release-dump }}. Schema version: ${{ github.event.inputs.schema-version }}. Data env: ${{ github.event.inputs.data-env }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
text: 'PROD full index from dump status: ${{ steps.indexdatadump.outcome }}. Using version: ${{ github.event.inputs.schema-version }}. From file: ${{ github.event.inputs.release-dump }}. Schema version: ${{ github.event.inputs.schema-version }}. Data env: ${{ github.event.inputs.data-env }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'

22 changes: 17 additions & 5 deletions .github/workflows/prod_manual_index.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,13 @@ name: Manual deploy to Prod
on:
workflow_dispatch:
inputs:
schema-version:
required: true
description: Schema version
type: choice
options:
- v1
- v2
release-tag:
type: string
required: true
Expand Down Expand Up @@ -49,8 +56,8 @@ jobs:
run: |
cd ./${{ github.event.inputs.release-tag }}
zip files.zip *.json
aws s3 cp files.zip s3://${{ secrets.ROR_DATA_S3_BUCKET_PROD }}/${{ github.event.inputs.release-tag }}/files.zip
aws s3 ls s3://${{ secrets.ROR_DATA_S3_BUCKET_PROD }}/${{ github.event.inputs.release-tag }}/files.zip
aws s3 cp files.zip s3://${{ secrets.ROR_DATA_S3_BUCKET_PROD }}/${{ github.event.inputs.release-tag }}-${{ github.event.inputs.schema-version }}/files.zip
aws s3 ls s3://${{ secrets.ROR_DATA_S3_BUCKET_PROD }}/${{ github.event.inputs.release-tag }}-${{ github.event.inputs.schema-version }}/files.zip
- name: Notify Slack
if: always()
uses: edge/simple-slack-notify@master
Expand All @@ -59,14 +66,19 @@ jobs:
with:
channel: '#ror-curation-releases'
color: 'good'
text: 'Copy status to PROD S3 Bucket: ${{ steps.copyfiles.outcome }}. From directory: ${{ github.event.inputs.release-tag }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
text: 'Copy status to PROD S3 Bucket: ${{ steps.copyfiles.outcome }}. Using version: ${{ github.event.inputs.schema-version }}. From directory: ${{ github.event.inputs.release-tag }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
- name: Index file
id: indexdata
run: |
cd .github/workflows
python -m pip install --upgrade pip
pip install requests==2.23.0
python index_files.py -u ${{ secrets.INDEX_PROD_API_URL }} -d ${{ github.event.inputs.release-tag }} -he ${{ secrets.INDEX_PROD_API_HEADERS }}
if [[ ${{ github.event.inputs.schema-version }} == 'v1' ]]; then
python index_files.py -u ${{ secrets.INDEX_PROD_API_URL_V1 }} -d ${{ github.event.inputs.directory-name }}-v1 -he ${{ secrets.INDEX_DEV_API_HEADERS }}
fi
if [[ ${{ github.event.inputs.schema-version }} == 'v2' ]]; then
python index_files.py -u ${{ secrets.INDEX_PROD_API_URL_V2 }} -d ${{ github.event.inputs.directory-name }}-v2 -he ${{ secrets.INDEX_DEV_API_HEADERS }}
fi
- name: Notify Slack
if: always()
uses: edge/simple-slack-notify@master
Expand All @@ -75,5 +87,5 @@ jobs:
with:
channel: '#ror-curation-releases'
color: 'good'
text: 'PROD index status to ES: ${{ steps.indexdata.outcome }}. From directory: ${{ github.event.inputs.release-tag }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
text: 'PROD index status to ES: ${{ steps.indexdata.outcome }}. Using version:. From directory: ${{ github.event.inputs.release-tag }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'

44 changes: 34 additions & 10 deletions .github/workflows/staging_manual_index.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,13 @@ name: Manual deploy to Staging
on:
workflow_dispatch:
inputs:
schema-version:
required: true
description: Schema version
type: choice
options:
- v1
- v2
directory-name:
type: string
required: true
Expand Down Expand Up @@ -49,6 +56,7 @@ jobs:
uses: actions/checkout@v2
with:
repository: ror-community/validation-suite
ref: schema-v2
path: validation-suite
- name: Validate files
id: validatefiles
Expand All @@ -58,11 +66,22 @@ jobs:
cd validation-suite
python -m pip install --upgrade pip
pip install -r requirements.txt
if [[ -f "../${{ github.event.inputs.directory-name }}/relationships.csv" ]]; then
python run_validations.py -i ../files -f ../${{ github.event.inputs.directory-name }}/relationships.csv -p ../files/ --no-geonames
else
python run_validations.py -i ../files --no-geonames
fi
if [[ ${{ github.event.inputs.schema-version }} == 'v1' ]]; then
curl https://raw.githubusercontent.com/ror-community/ror-schema/master/ror_schema.json -o ror_schema.json
if [[ -f "../${{github.event.pull_request.head.ref}}/relationships.csv" ]]; then
python run_validations.py -i ../files -v 1 -s ror_schema.json -f ../${{github.event.pull_request.head.ref}}/relationships.csv -p ../files/ --no-geonames
else
python run_validations.py -i ../files -v 1 -s ror_schema.json --no-geonames
fi
fi
if [[ ${{ github.event.inputs.schema-version }} == 'v2' ]]; then
curl https://raw.githubusercontent.com/ror-community/ror-schema/master/ror_schema_v2_0.json -o ror_schema_v2_0.json
if [[ -f "../${{github.event.pull_request.head.ref}}/relationships.csv" ]]; then
python run_validations.py -i ../files -v 2 -s ror_schema_v2_0.json -f ../${{github.event.pull_request.head.ref}}/relationships.csv -p ../files/ --no-geonames
else
python run_validations.py -i ../files -v 2 -s ror_schema_v2_0.json --no-geonames
fi
fi
- name: Notify Slack
if: always()
uses: edge/simple-slack-notify@master
Expand All @@ -71,7 +90,7 @@ jobs:
with:
channel: '#ror-curation-releases'
color: 'good'
text: 'Validation status during STAGING manual index: ${{ steps.validatefiles.outcome }}. In directory: ${{ github.event.inputs.directory-name }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
text: 'Validation status during STAGING manual index: ${{ steps.validatefiles.outcome }}. Using version: ${{ github.event.inputs.schema-version }}. In directory: ${{ github.event.inputs.directory-name }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
copy-files:
runs-on: ubuntu-latest
if: github.event.ref == 'refs/heads/staging'
Expand All @@ -89,8 +108,8 @@ jobs:
run: |
cd ./${{ github.event.inputs.directory-name }}
zip files.zip *.json
aws s3 cp files.zip s3://${{ secrets.ROR_DATA_S3_BUCKET_STAGING }}/${{ github.event.inputs.directory-name }}/files.zip
aws s3 ls s3://${{ secrets.ROR_DATA_S3_BUCKET_STAGING }}/${{ github.event.inputs.directory-name }}/files.zip
aws s3 cp files.zip s3://${{ secrets.ROR_DATA_S3_BUCKET_STAGING }}/${{ github.event.inputs.directory-name }}-${{ github.event.inputs.schema-version }}/files.zip
aws s3 ls s3://${{ secrets.ROR_DATA_S3_BUCKET_STAGING }}/${{ github.event.inputs.directory-name }}-${{ github.event.inputs.schema-version }}/files.zip
- name: Notify Slack
if: always()
uses: edge/simple-slack-notify@master
Expand All @@ -106,7 +125,12 @@ jobs:
cd .github/workflows
python -m pip install --upgrade pip
pip install requests==2.23.0
python index_files.py -u ${{ secrets.INDEX_STAGING_API_URL }} -d ${{ github.event.inputs.directory-name }} -he ${{ secrets.INDEX_STAGING_API_HEADERS }}
if [[ ${{ github.event.inputs.schema-version }} == 'v1' ]]; then
python index_files.py -u ${{ secrets.INDEX_STAGING_API_URLL_V1 }} -d ${{ github.event.inputs.directory-name }}-v1 -he ${{ secrets.INDEX_STAGING_API_HEADERS }}
fi
if [[ ${{ github.event.inputs.schema-version }} == 'v2' ]]; then
python index_files.py -u ${{ secrets.INDEX_STAGING_API_URL_V2 }} -d ${{ github.event.inputs.directory-name }}-v2 -he ${{ secrets.INDEX_STAGING_API_HEADERS }}
fi
- name: Notify Slack
if: always()
uses: edge/simple-slack-notify@master
Expand All @@ -115,5 +139,5 @@ jobs:
with:
channel: '#ror-curation-releases'
color: 'good'
text: 'STAGING index status to ES: ${{ steps.indexdata.outcome }}. From directory: ${{ github.event.inputs.directory-name }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
text: 'STAGING index status to ES: ${{ steps.indexdata.outcome }}. Using version ${{ github.event.inputs.schema-version }}. From directory: ${{ github.event.inputs.directory-name }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'

12 changes: 6 additions & 6 deletions .github/workflows/staging_merge.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Deploy to Staging
name: Deploy to Staging v2 only
on:
push:
branches:
Expand Down Expand Up @@ -39,8 +39,8 @@ jobs:
run: |
cd ./${{ env.release_branch }}
zip files.zip *.json
aws s3 cp files.zip s3://${{ secrets.ROR_DATA_S3_BUCKET_STAGING }}/${{ env.release_branch }}/files.zip
aws s3 ls s3://${{ secrets.ROR_DATA_S3_BUCKET_STAGING }}/${{ env.release_branch }}/files.zip
aws s3 cp files.zip s3://${{ secrets.ROR_DATA_S3_BUCKET_STAGING }}/${{ env.release_branch }}-v2/files.zip
aws s3 ls s3://${{ secrets.ROR_DATA_S3_BUCKET_STAGING }}/${{ env.release_branch }}-v2/files.zip
- name: Notify Slack
if: always()
uses: edge/simple-slack-notify@master
Expand All @@ -49,14 +49,14 @@ jobs:
with:
channel: '#ror-curation-releases'
color: 'good'
text: 'Copy status to staging S3 Bucket: ${{ steps.copyfiles.outcome }}. On branch: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/tree/${env.GITHUB_REF_NAME}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
text: 'Copy status to staging S3 Bucket: ${{ steps.copyfiles.outcome }}. Using v2. On branch: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/tree/${env.GITHUB_REF_NAME}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
- name: Index file
id: indexdata
run: |
cd .github/workflows
python -m pip install --upgrade pip
pip install requests==2.23.0
python index_files.py -u ${{ secrets.INDEX_STAGING_API_URL }} -d ${{ env.release_branch }} -he ${{ secrets.INDEX_STAGING_API_HEADERS }}
python index_files.py -u ${{ secrets.INDEX_STAGING_API_URL_V2 }} -d ${{ env.release_branch }}-v2 -he ${{ secrets.INDEX_STAGING_API_HEADERS }}
- name: Notify Slack
if: always()
uses: edge/simple-slack-notify@master
Expand All @@ -65,5 +65,5 @@ jobs:
with:
channel: '#ror-curation-releases'
color: 'good'
text: 'Staging index status: ${{ steps.indexdata.outcome }}. On branch: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/tree/${env.GITHUB_REF_NAME}. Directory: ${{ env.release_branch }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
text: 'Staging index status: ${{ steps.indexdata.outcome }}. Using v2. On branch: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/tree/${env.GITHUB_REF_NAME}. Directory: ${{ env.release_branch }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'

8 changes: 5 additions & 3 deletions .github/workflows/staging_pull_request.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ jobs:
uses: actions/checkout@v2
with:
repository: ror-community/validation-suite
ref: schema-v2
path: validation-suite
- name: Validate files
id: validatefiles
Expand All @@ -58,10 +59,11 @@ jobs:
cd validation-suite
python -m pip install --upgrade pip
pip install -r requirements.txt
curl https://raw.githubusercontent.com/ror-community/ror-schema/master/ror_schema_v2_0.json -o ror_schema_v2_0.json
if [[ -f "../${{github.event.pull_request.head.ref}}/relationships.csv" ]]; then
python run_validations.py -i ../files -f ../${{github.event.pull_request.head.ref}}/relationships.csv -p ../files/ --no-geonames
python run_validations.py -i ../files -v 2 -s ror_schema_v2_0.json -f ../${{github.event.pull_request.head.ref}}/relationships.csv -p ../files/ --no-geonames
else
python run_validations.py -i ../files --no-geonames
python run_validations.py -i ../files -v 2 -s ror_schema_v2_0.json --no-geonames
fi
- name: Notify Slack
if: always()
Expand All @@ -71,5 +73,5 @@ jobs:
with:
channel: '#ror-curation-releases'
color: 'good'
text: 'Validation status during PR to STAGING: ${{ steps.validatefiles.outcome }}. On branch: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/tree/${env.GITHUB_HEAD_REF}. In directory: ${env.GITHUB_HEAD_REF}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
text: 'Validation status during PR to STAGING: ${{ steps.validatefiles.outcome }}. Using v2. On branch: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/tree/${env.GITHUB_HEAD_REF}. In directory: ${env.GITHUB_HEAD_REF}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'

Loading

0 comments on commit 3bc3bb7

Please sign in to comment.