ror-community · adambuttrick · Apr 11, 2024 · Apr 9, 2024 · Apr 10, 2024 · Apr 11, 2024
diff --git a/.github/workflows/generate_dump.yml b/.github/workflows/generate_dump.yml
@@ -2,6 +2,13 @@ name: Create data dump
 on:
   workflow_dispatch:
     inputs:
+      schema-version:
+        required: true
+        description: Release schema version
+        type: choice
+        options:
+        - v1
+        - v2
       new-release:
         type: string
         description: Name of the directory that the new release is located in
@@ -25,7 +32,7 @@ jobs:
       - name: checkout ror data repo
         uses: actions/checkout@v2
         with:
-          repository: ror-community/ror-data
+          repository: ror-community/ror-data-test
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
           path: ./ror-data
       - name: copy previous data dump file
@@ -37,6 +44,7 @@ jobs:
         with:
           repository: ror-community/curation_ops
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+          ref: v2-crosswalk
           path: ./curation_ops
       - name: Set up Python environment
         uses: actions/setup-python@v2
@@ -46,7 +54,12 @@ jobs:
         id: gendumpscript
         run: |
           cd ./curation_ops/generate_dump/
-          python generate_dump.py -r ${{github.event.inputs.new-release}} -e ${{github.event.inputs.prev-release}} -i '../../ror-records' -o '../../ror-records'
+          if [[ ${{ github.event.inputs.schema-version }} == 'v1' ]]; then
+            python generate_dump.py -r ${{github.event.inputs.new-release}} -e ${{github.event.inputs.prev-release}} -i '../../ror-records' -o '../../ror-records' -v 1
+          fi
+          if [[ ${{ github.event.inputs.schema-version }} == 'v2' ]]; then
+            python generate_dump.py -r ${{github.event.inputs.new-release}} -e ${{github.event.inputs.prev-release}} -i '../../ror-records' -o '../../ror-records' -v 2
+          fi
       - name: cat error file
         if: ${{ steps.gendumpscript.outcome != 'success'}}
         run: |
@@ -61,11 +74,20 @@ jobs:
         if: ${{ steps.copynewdump.outcome == 'success'}}
         run: |
           cd ./ror-data
-          git config --local user.email "[email protected]"
-          git config --local user.name "GitHub Action"
+          git config --local user.email "[email protected]"
+          git config --local user.name "ror-bot"
           git add *.zip
           git commit -m "add new data dump file"
-          git push origin main
+          git push origin
+      - name: commit changed files
+        if: ${{ steps.commitdumpfile.outcome == 'success'}} && ${{ github.event.inputs.schema-version }} == 'v2'
+        run: |
+          cd ./ror-records
+          git config --local user.email "[email protected]"
+          git config --local user.name "ror-bot"
+          git add ${{github.event.inputs.new-release}}/v1/
+          git commit -m "add generated v1 files"
+          git push origin ${{ github.event.ref }}
       - name: Notify Slack
         if: always()
         uses: edge/simple-slack-notify@master
@@ -74,4 +96,4 @@ jobs:
         with:
           channel: '#ror-curation-releases'
           color: 'good'
-          text: 'PROD Data dump ${{github.event.inputs.new-release}} generation status:  ${{ steps.commitdumpfile.outcome }} Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+          text: 'PROD Data dump ${{github.event.inputs.new-release}} generation status:  ${{ steps.commitdumpfile.outcome }}. Using base version: ${{ github.event.inputs.schema-version }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
diff --git a/.github/workflows/main_release.yml b/.github/workflows/main_release.yml
@@ -1,4 +1,4 @@
-name: Deploy to Prod
+name: Deploy to Prod v2 only
 on:
   release:
     types: [published]
@@ -50,8 +50,8 @@ jobs:
         run: |
           cd ./${{ github.event.release.tag_name }}
           zip files.zip *.json
-          aws s3 cp files.zip s3://${{ secrets.ROR_DATA_S3_BUCKET_PROD }}/${{ github.event.release.tag_name }}/files.zip
-          aws s3 ls s3://${{ secrets.ROR_DATA_S3_BUCKET_PROD }}/${{ github.event.release.tag_name }}/files.zip
+          aws s3 cp files.zip s3://${{ secrets.ROR_DATA_S3_BUCKET_PROD }}/${{ github.event.release.tag_name }}-v2/files.zip
+          aws s3 ls s3://${{ secrets.ROR_DATA_S3_BUCKET_PROD }}/${{ github.event.release.tag_name }}-v2/files.zip
       - name: Notify Slack
         if: always()
         uses: edge/simple-slack-notify@master
@@ -60,14 +60,14 @@ jobs:
         with:
           channel: '#ror-curation-releases'
           color: 'good'
-          text: 'Copy status to PROD S3 Bucket:  ${{ steps.copyfiles.outcome }}.  From directory: ${{ github.event.release.tag_name }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+          text: 'Copy status to PROD S3 Bucket:  ${{ steps.copyfiles.outcome }}. Using version 2. From directory: ${{ github.event.release.tag_name }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
       - name: Index data
         id: indexdata
         run: |
           cd .github/workflows
           python -m pip install --upgrade pip
           pip install requests==2.23.0
-          python index_files.py -u ${{ secrets.INDEX_PROD_API_URL }} -d ${{ github.event.release.tag_name }} -he ${{ secrets.INDEX_PROD_API_HEADERS }}
+          python index_files.py -u ${{ secrets.INDEX_PROD_API_URL_V2 }} -d ${{ github.event.release.tag_name }}-v2 -he ${{ secrets.INDEX_PROD_API_HEADERS }}
       - name: Notify Slack
         if: always()
         uses: edge/simple-slack-notify@master
@@ -76,4 +76,4 @@ jobs:
         with:
           channel: '#ror-curation-releases'
           color: 'good'
-          text: 'PROD index status: ${{ steps.indexdata.outcome }}. From directory: ${{ github.event.release.tag_name }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+          text: 'PROD index status: ${{ steps.indexdata.outcome }}. Using version 2. From directory: ${{ github.event.release.tag_name }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
diff --git a/.github/workflows/prod_index_dump.yml b/.github/workflows/prod_index_dump.yml
@@ -74,5 +74,5 @@ jobs:
         with:
           channel: '#ror-curation-releases'
           color: 'good'
-          text: 'PROD full index from dump status: ${{ steps.indexdatadump.outcome }}. From file: ${{ github.event.inputs.release-dump }}. Schema version: ${{ github.event.inputs.schema-version }}. Data env: ${{ github.event.inputs.data-env }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+          text: 'PROD full index from dump status: ${{ steps.indexdatadump.outcome }}. Using version: ${{ github.event.inputs.schema-version }}. From file: ${{ github.event.inputs.release-dump }}. Schema version: ${{ github.event.inputs.schema-version }}. Data env: ${{ github.event.inputs.data-env }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
 
diff --git a/.github/workflows/prod_manual_index.yml b/.github/workflows/prod_manual_index.yml
@@ -2,6 +2,13 @@ name: Manual deploy to Prod
 on:
   workflow_dispatch:
     inputs:
+      schema-version:
+        required: true
+        description: Schema version
+        type: choice
+        options:
+        - v1
+        - v2
       release-tag:
         type: string
         required: true
@@ -49,8 +56,8 @@ jobs:
         run: |
           cd ./${{ github.event.inputs.release-tag }}
           zip files.zip *.json
-          aws s3 cp files.zip s3://${{ secrets.ROR_DATA_S3_BUCKET_PROD }}/${{ github.event.inputs.release-tag }}/files.zip
-          aws s3 ls s3://${{ secrets.ROR_DATA_S3_BUCKET_PROD }}/${{ github.event.inputs.release-tag }}/files.zip
+          aws s3 cp files.zip s3://${{ secrets.ROR_DATA_S3_BUCKET_PROD }}/${{ github.event.inputs.release-tag }}-${{ github.event.inputs.schema-version }}/files.zip
+          aws s3 ls s3://${{ secrets.ROR_DATA_S3_BUCKET_PROD }}/${{ github.event.inputs.release-tag }}-${{ github.event.inputs.schema-version }}/files.zip
       - name: Notify Slack
         if: always()
         uses: edge/simple-slack-notify@master
@@ -59,14 +66,19 @@ jobs:
         with:
           channel: '#ror-curation-releases'
           color: 'good'
-          text: 'Copy status to PROD S3 Bucket:  ${{ steps.copyfiles.outcome }}.  From directory: ${{ github.event.inputs.release-tag }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+          text: 'Copy status to PROD S3 Bucket:  ${{ steps.copyfiles.outcome }}.  Using version: ${{ github.event.inputs.schema-version }}. From directory: ${{ github.event.inputs.release-tag }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
       - name: Index file
         id: indexdata
         run: |
           cd .github/workflows
           python -m pip install --upgrade pip
           pip install requests==2.23.0
-          python index_files.py -u ${{ secrets.INDEX_PROD_API_URL }} -d ${{ github.event.inputs.release-tag }} -he ${{ secrets.INDEX_PROD_API_HEADERS }}
+          if [[ ${{ github.event.inputs.schema-version }} == 'v1' ]]; then
+            python index_files.py -u ${{ secrets.INDEX_PROD_API_URL_V1 }} -d ${{ github.event.inputs.directory-name }}-v1 -he ${{ secrets.INDEX_DEV_API_HEADERS }}
+          fi
+          if [[ ${{ github.event.inputs.schema-version }} == 'v2' ]]; then
+            python index_files.py -u ${{ secrets.INDEX_PROD_API_URL_V2 }} -d ${{ github.event.inputs.directory-name }}-v2 -he ${{ secrets.INDEX_DEV_API_HEADERS }}
+          fi
       - name: Notify Slack
         if: always()
         uses: edge/simple-slack-notify@master
@@ -75,5 +87,5 @@ jobs:
         with:
           channel: '#ror-curation-releases'
           color: 'good'
-          text: 'PROD index status to ES: ${{ steps.indexdata.outcome }}. From directory: ${{ github.event.inputs.release-tag }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+          text: 'PROD index status to ES: ${{ steps.indexdata.outcome }}. Using version:. From directory: ${{ github.event.inputs.release-tag }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
 
diff --git a/.github/workflows/staging_manual_index.yml b/.github/workflows/staging_manual_index.yml
@@ -2,6 +2,13 @@ name: Manual deploy to Staging
 on:
   workflow_dispatch:
     inputs:
+      schema-version:
+            required: true
+            description: Schema version
+            type: choice
+            options:
+            - v1
+            - v2
       directory-name:
         type: string
         required: true
@@ -49,6 +56,7 @@ jobs:
         uses: actions/checkout@v2
         with:
           repository: ror-community/validation-suite
+          ref: schema-v2
           path: validation-suite
       - name: Validate files
         id: validatefiles
@@ -58,11 +66,22 @@ jobs:
           cd validation-suite
           python -m pip install --upgrade pip
           pip install -r requirements.txt
-          if [[ -f "../${{ github.event.inputs.directory-name }}/relationships.csv" ]]; then
-              python run_validations.py -i ../files -f ../${{ github.event.inputs.directory-name }}/relationships.csv -p ../files/ --no-geonames
-            else
-              python run_validations.py -i ../files --no-geonames
-            fi
+          if [[ ${{ github.event.inputs.schema-version }} == 'v1' ]]; then
+            curl https://raw.githubusercontent.com/ror-community/ror-schema/master/ror_schema.json -o ror_schema.json
+            if [[ -f "../${{github.event.pull_request.head.ref}}/relationships.csv" ]]; then
+                python run_validations.py -i ../files -v 1 -s ror_schema.json -f ../${{github.event.pull_request.head.ref}}/relationships.csv -p ../files/ --no-geonames
+              else
+                python run_validations.py -i ../files -v 1 -s ror_schema.json --no-geonames
+              fi
+          fi
+          if [[ ${{ github.event.inputs.schema-version }} == 'v2' ]]; then
+            curl https://raw.githubusercontent.com/ror-community/ror-schema/master/ror_schema_v2_0.json -o ror_schema_v2_0.json
+            if [[ -f "../${{github.event.pull_request.head.ref}}/relationships.csv" ]]; then
+                python run_validations.py -i ../files -v 2 -s ror_schema_v2_0.json -f ../${{github.event.pull_request.head.ref}}/relationships.csv -p ../files/ --no-geonames
+              else
+                python run_validations.py -i ../files -v 2 -s ror_schema_v2_0.json --no-geonames
+              fi
+          fi
       - name: Notify Slack
         if: always()
         uses: edge/simple-slack-notify@master
@@ -71,7 +90,7 @@ jobs:
         with:
           channel: '#ror-curation-releases'
           color: 'good'
-          text: 'Validation status during STAGING manual index:  ${{ steps.validatefiles.outcome }}. In directory: ${{ github.event.inputs.directory-name }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+          text: 'Validation status during STAGING manual index:  ${{ steps.validatefiles.outcome }}. Using version: ${{ github.event.inputs.schema-version }}. In directory: ${{ github.event.inputs.directory-name }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
   copy-files:
     runs-on: ubuntu-latest
     if: github.event.ref == 'refs/heads/staging'
@@ -89,8 +108,8 @@ jobs:
         run: |
           cd ./${{ github.event.inputs.directory-name }}
           zip files.zip *.json
-          aws s3 cp files.zip s3://${{ secrets.ROR_DATA_S3_BUCKET_STAGING }}/${{ github.event.inputs.directory-name }}/files.zip
-          aws s3 ls s3://${{ secrets.ROR_DATA_S3_BUCKET_STAGING }}/${{ github.event.inputs.directory-name }}/files.zip
+          aws s3 cp files.zip s3://${{ secrets.ROR_DATA_S3_BUCKET_STAGING }}/${{ github.event.inputs.directory-name }}-${{ github.event.inputs.schema-version }}/files.zip
+          aws s3 ls s3://${{ secrets.ROR_DATA_S3_BUCKET_STAGING }}/${{ github.event.inputs.directory-name }}-${{ github.event.inputs.schema-version }}/files.zip
       - name: Notify Slack
         if: always()
         uses: edge/simple-slack-notify@master
@@ -106,7 +125,12 @@ jobs:
           cd .github/workflows
           python -m pip install --upgrade pip
           pip install requests==2.23.0
-          python index_files.py -u ${{ secrets.INDEX_STAGING_API_URL }} -d ${{ github.event.inputs.directory-name }} -he ${{ secrets.INDEX_STAGING_API_HEADERS }}
+          if [[ ${{ github.event.inputs.schema-version }} == 'v1' ]]; then
+            python index_files.py -u ${{ secrets.INDEX_STAGING_API_URLL_V1 }} -d ${{ github.event.inputs.directory-name }}-v1 -he ${{ secrets.INDEX_STAGING_API_HEADERS }}
+          fi
+          if [[ ${{ github.event.inputs.schema-version }} == 'v2' ]]; then
+            python index_files.py -u ${{ secrets.INDEX_STAGING_API_URL_V2 }} -d ${{ github.event.inputs.directory-name }}-v2 -he ${{ secrets.INDEX_STAGING_API_HEADERS }}
+          fi
       - name: Notify Slack
         if: always()
         uses: edge/simple-slack-notify@master
@@ -115,5 +139,5 @@ jobs:
         with:
           channel: '#ror-curation-releases'
           color: 'good'
-          text: 'STAGING index status to ES: ${{ steps.indexdata.outcome }}. From directory: ${{ github.event.inputs.directory-name }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+          text: 'STAGING index status to ES: ${{ steps.indexdata.outcome }}. Using version ${{ github.event.inputs.schema-version }}. From directory: ${{ github.event.inputs.directory-name }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
 
diff --git a/.github/workflows/staging_merge.yml b/.github/workflows/staging_merge.yml
@@ -1,4 +1,4 @@
-name: Deploy to Staging
+name: Deploy to Staging v2 only
 on:
   push:
     branches:
@@ -39,8 +39,8 @@ jobs:
         run: |
           cd ./${{ env.release_branch }}
           zip files.zip *.json
-          aws s3 cp files.zip s3://${{ secrets.ROR_DATA_S3_BUCKET_STAGING }}/${{ env.release_branch }}/files.zip
-          aws s3 ls s3://${{ secrets.ROR_DATA_S3_BUCKET_STAGING }}/${{ env.release_branch }}/files.zip
+          aws s3 cp files.zip s3://${{ secrets.ROR_DATA_S3_BUCKET_STAGING }}/${{ env.release_branch }}-v2/files.zip
+          aws s3 ls s3://${{ secrets.ROR_DATA_S3_BUCKET_STAGING }}/${{ env.release_branch }}-v2/files.zip
       - name: Notify Slack
         if: always()
         uses: edge/simple-slack-notify@master
@@ -49,14 +49,14 @@ jobs:
         with:
           channel: '#ror-curation-releases'
           color: 'good'
-          text: 'Copy status to staging S3 Bucket:  ${{ steps.copyfiles.outcome }}.  On branch: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/tree/${env.GITHUB_REF_NAME}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+          text: 'Copy status to staging S3 Bucket:  ${{ steps.copyfiles.outcome }}. Using v2. On branch: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/tree/${env.GITHUB_REF_NAME}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
       - name: Index file
         id: indexdata
         run: |
           cd .github/workflows
           python -m pip install --upgrade pip
           pip install requests==2.23.0
-          python index_files.py -u ${{ secrets.INDEX_STAGING_API_URL }} -d ${{ env.release_branch }} -he ${{ secrets.INDEX_STAGING_API_HEADERS }}
+          python index_files.py -u ${{ secrets.INDEX_STAGING_API_URL_V2 }} -d ${{ env.release_branch }}-v2 -he ${{ secrets.INDEX_STAGING_API_HEADERS }}
       - name: Notify Slack
         if: always()
         uses: edge/simple-slack-notify@master
@@ -65,5 +65,5 @@ jobs:
         with:
           channel: '#ror-curation-releases'
           color: 'good'
-          text: 'Staging index status: ${{ steps.indexdata.outcome }}. On branch: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/tree/${env.GITHUB_REF_NAME}. Directory: ${{ env.release_branch }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+          text: 'Staging index status: ${{ steps.indexdata.outcome }}. Using v2. On branch: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/tree/${env.GITHUB_REF_NAME}. Directory: ${{ env.release_branch }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
 
diff --git a/.github/workflows/staging_pull_request.yml b/.github/workflows/staging_pull_request.yml
@@ -49,6 +49,7 @@ jobs:
         uses: actions/checkout@v2
         with:
           repository: ror-community/validation-suite
+          ref: schema-v2
           path: validation-suite
       - name: Validate files
         id: validatefiles
@@ -58,10 +59,11 @@ jobs:
           cd validation-suite
           python -m pip install --upgrade pip
           pip install -r requirements.txt
+          curl https://raw.githubusercontent.com/ror-community/ror-schema/master/ror_schema_v2_0.json -o ror_schema_v2_0.json
           if [[ -f "../${{github.event.pull_request.head.ref}}/relationships.csv" ]]; then
-              python run_validations.py -i ../files -f ../${{github.event.pull_request.head.ref}}/relationships.csv -p ../files/ --no-geonames
+              python run_validations.py -i ../files -v 2 -s ror_schema_v2_0.json -f ../${{github.event.pull_request.head.ref}}/relationships.csv -p ../files/ --no-geonames
             else
-              python run_validations.py -i ../files --no-geonames
+              python run_validations.py -i ../files -v 2 -s ror_schema_v2_0.json --no-geonames
             fi
       - name: Notify Slack
         if: always()
@@ -71,5 +73,5 @@ jobs:
         with:
           channel: '#ror-curation-releases'
           color: 'good'
-          text: 'Validation status during PR to STAGING:  ${{ steps.validatefiles.outcome }}. On branch: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/tree/${env.GITHUB_HEAD_REF}. In directory: ${env.GITHUB_HEAD_REF}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+          text: 'Validation status during PR to STAGING:  ${{ steps.validatefiles.outcome }}. Using v2. On branch: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/tree/${env.GITHUB_HEAD_REF}. In directory: ${env.GITHUB_HEAD_REF}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'