Merge pull request #210 from ror-community/dev

v1.43

.github/workflows/dev_index_dump.yml

@@ -8,7 +8,6 @@ on:
       schema-version:
         required: true
         description: Schema version to index
-        default: 'test'
         type: choice
         options:
         - v1
@@ -53,18 +52,19 @@ jobs:
     if: github.event.ref == 'refs/heads/dev'
     needs: check_permission
     steps:
+      - name: checkout
+        uses: actions/checkout@v2
       - name: Index dump file
         id: indexdatadump
         run: |
           cd .github/workflows
           python -m pip install --upgrade pip
           pip install requests==2.23.0
-          if [[ ${{ github.event.inputs.data-env }} == "v1" ]]; then
+          if [[ ${{ github.event.inputs.schema-version }} == "v1" ]]; then
             python index_dump.py -u ${{ secrets.INDEX_DUMP_DEV_API_URL }} -f ${{ github.event.inputs.release-dump }} -e  ${{ github.event.inputs.data-env }} -he ${{ secrets.INDEX_DEV_API_HEADERS }}
           else
             python index_dump.py -u ${{ secrets.INDEX_DUMP_DEV_API_URL_V2 }} -f ${{ github.event.inputs.release-dump }} -e  ${{ github.event.inputs.data-env }} -he ${{ secrets.INDEX_DEV_API_HEADERS }}
           fi
-
       - name: Notify Slack
         if: always()
         uses: edge/simple-slack-notify@master
@@ -73,5 +73,5 @@ jobs:
         with:
           channel: '#ror-curation-releases'
           color: 'good'
-          text: 'DEV full index from dump status: ${{ steps.indexdata.outcome }}. From file: ${{ github.event.inputs.release-dump }}. Schema version: ${{ github.event.inputs.data-env }}. Data env: ${{ github.event.inputs.release-dump }}Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+          text: 'DEV full index from dump status: ${{ steps.indexdatadump.outcome }}. From file: ${{ github.event.inputs.release-dump }}. Schema version: ${{ github.event.inputs.schema-version }}. Data env: ${{ github.event.inputs.data-env }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
 

.github/workflows/dev_manual_index.yml

@@ -2,6 +2,13 @@ name: DEV manual deploy from dev branch
 on:
   workflow_dispatch:
     inputs:
+      schema-version:
+            required: true
+            description: Schema version
+            type: choice
+            options:
+            - v1
+            - v2
       directory-name:
         type: string
         required: true
@@ -15,7 +22,7 @@ env:
 jobs:
   check_permission:
     runs-on: ubuntu-latest
-    if: github.event.ref == 'refs/heads/dev'
+    #if: github.event.ref == 'refs/heads/dev'
     steps:
       - name: Get Permission
         uses: octokit/[email protected]
@@ -34,7 +41,7 @@ jobs:
           fi
   check_validation:
     runs-on: ubuntu-latest
-    if: github.event.ref == 'refs/heads/dev'
+    #if: github.event.ref == 'refs/heads/dev'
     needs: check_permission
     steps:
       - name: checkout
@@ -49,6 +56,7 @@ jobs:
         uses: actions/checkout@v2
         with:
           repository: ror-community/validation-suite
+          ref: schema-v2
           path: validation-suite
       - name: Validate files
         id: validatefiles
@@ -58,11 +66,22 @@ jobs:
           cd validation-suite
           python -m pip install --upgrade pip
           pip install -r requirements.txt
-          if [[ -f "../${{ github.event.inputs.directory-name }}/relationships.csv" ]]; then
-              python run_validations.py -i ../files -f ../${{ github.event.inputs.directory-name }}/relationships.csv -p ../files/ --no-geonames
-            else
-              python run_validations.py -i ../files --no-geonames
-            fi
+          if [[ ${{ github.event.inputs.schema-version }} == 'v1' ]]; then
+            curl https://raw.githubusercontent.com/ror-community/ror-schema/schema-v2/ror_schema.json -o ror_schema.json
+            if [[ -f "../${{github.event.pull_request.head.ref}}/relationships.csv" ]]; then
+                python run_validations.py -i ../files -v 1 -s ror_schema.json -f ../${{github.event.pull_request.head.ref}}/relationships.csv -p ../files/ --no-geonames
+              else
+                python run_validations.py -i ../files -v 1 -s ror_schema.json --no-geonames
+              fi
+          fi
+          if [[ ${{ github.event.inputs.schema-version }} == 'v2' ]]; then
+            curl https://raw.githubusercontent.com/ror-community/ror-schema/schema-v2/ror_schema_v2_0.json -o ror_schema_v2_0.json
+            if [[ -f "../${{github.event.pull_request.head.ref}}/relationships.csv" ]]; then
+                python run_validations.py -i ../files -v 2 -s ror_schema_v2_0.json -f ../${{github.event.pull_request.head.ref}}/relationships.csv -p ../files/ --no-geonames
+              else
+                python run_validations.py -i ../files -v 2 -s ror_schema_v2_0.json --no-geonames
+              fi
+          fi
       - name: Notify Slack
         if: always()
         uses: edge/simple-slack-notify@master
@@ -74,7 +93,7 @@ jobs:
           text: 'Validation status during DEV manual index:  ${{ steps.validatefiles.outcome }}. In directory: ${{ github.event.inputs.directory-name }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
   copy-files:
     runs-on: ubuntu-latest
-    if: github.event.ref == 'refs/heads/dev'
+    #if: github.event.ref == 'refs/heads/dev'
     needs: check_validation
     steps:
       - name: checkout
@@ -89,8 +108,8 @@ jobs:
         run: |
           cd ./${{ github.event.inputs.directory-name }}
           zip files.zip *.json
-          aws s3 cp files.zip s3://${{ secrets.ROR_DATA_S3_BUCKET_DEV }}/${{ github.event.inputs.directory-name }}/files.zip
-          aws s3 ls s3://${{ secrets.ROR_DATA_S3_BUCKET_DEV }}/${{ github.event.inputs.directory-name }}/files.zip
+          aws s3 cp files.zip s3://${{ secrets.ROR_DATA_S3_BUCKET_DEV }}/${{ github.event.inputs.directory-name }}-${{ github.event.inputs.schema-version }}/files.zip
+          aws s3 ls s3://${{ secrets.ROR_DATA_S3_BUCKET_DEV }}/${{ github.event.inputs.directory-name }}-${{ github.event.inputs.schema-version }}/files.zip
       - name: Notify Slack
         if: always()
         uses: edge/simple-slack-notify@master
@@ -106,7 +125,13 @@ jobs:
           cd .github/workflows
           python -m pip install --upgrade pip
           pip install requests==2.23.0
-          python index_files.py -u ${{ secrets.INDEX_DEV_API_URL }} -d ${{ github.event.inputs.directory-name }} -he ${{ secrets.INDEX_DEV_API_HEADERS }}
+          if [[ ${{ github.event.inputs.schema-version }} == 'v1' ]]; then
+            python index_files.py -u ${{ secrets.INDEX_DEV_API_URL_V1 }} -d ${{ github.event.inputs.directory-name }}-v1 -he ${{ secrets.INDEX_DEV_API_HEADERS }}
+          fi
+          if [[ ${{ github.event.inputs.schema-version }} == 'v2' ]]; then
+            python index_files.py -u ${{ secrets.INDEX_DEV_API_URL_V2 }} -d ${{ github.event.inputs.directory-name }}-v2 -he ${{ secrets.INDEX_DEV_API_HEADERS }}
+          fi
+
       - name: Notify Slack
         if: always()
         uses: edge/simple-slack-notify@master
@@ -115,5 +140,5 @@ jobs:
         with:
           channel: '#ror-curation-releases'
           color: 'good'
-          text: 'DEV index status: ${{ steps.indexdata.outcome }}. From directory: ${{ github.event.inputs.directory-name }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+          text: 'DEV index status: ${{ steps.indexdata.outcome }}. Using version ${{ github.event.inputs.schema-version }}. From directory: ${{ github.event.inputs.directory-name }}-${{ github.event.inputs.schema-version }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
 

.github/workflows/dev_merge.yml

@@ -39,8 +39,8 @@ jobs:
         run: |
           cd ./${{ env.release_branch }}
           zip files.zip *.json
-          aws s3 cp files.zip s3://${{ secrets.ROR_DATA_S3_BUCKET_DEV}}/${{ env.release_branch }}/files.zip
-          aws s3 ls s3://${{ secrets.ROR_DATA_S3_BUCKET_DEV }}/${{ env.release_branch }}/files.zip
+          aws s3 cp files.zip s3://${{ secrets.ROR_DATA_S3_BUCKET_DEV}}/${{ env.release_branch }}-v2/files.zip
+          aws s3 ls s3://${{ secrets.ROR_DATA_S3_BUCKET_DEV }}/${{ env.release_branch }}-v2/files.zip
       - name: Notify Slack
         if: always()
         uses: edge/simple-slack-notify@master
@@ -49,14 +49,14 @@ jobs:
         with:
           channel: '#ror-curation-releases'
           color: 'good'
-          text: 'Copy status to DEV S3 Bucket:  ${{ steps.copyfiles.outcome }}.  On branch: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/tree/${env.GITHUB_REF_NAME}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+          text: 'Copy status to DEV S3 Bucket:  ${{ steps.copyfiles.outcome }}. Using v2. On branch: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/tree/${env.GITHUB_REF_NAME}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
       - name: Index file
         id: indexdata
         run: |
           cd .github/workflows
           python -m pip install --upgrade pip
           pip install requests==2.23.0
-          python index_files.py -u ${{ secrets.INDEX_DEV_API_URL }} -d ${{ env.release_branch }} -he ${{ secrets.INDEX_DEV_API_HEADERS }}
+          python index_files.py -u ${{ secrets.INDEX_DEV_API_URL_v2 }} -d ${{ env.release_branch }}-v2 -he ${{ secrets.INDEX_DEV_API_HEADERS }}
       - name: Notify Slack
         if: always()
         uses: edge/simple-slack-notify@master
@@ -65,5 +65,5 @@ jobs:
         with:
           channel: '#ror-curation-releases'
           color: 'good'
-          text: 'DEV index status: ${{ steps.indexdata.outcome }}. On branch: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/tree/${env.GITHUB_REF_NAME}. Directory: ${{ env.release_branch }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+          text: 'DEV index status: ${{ steps.indexdata.outcome }}. Using v2. On branch: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/tree/${env.GITHUB_REF_NAME}. Directory: ${{ env.release_branch }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
 

.github/workflows/dev_pull_request.yml

@@ -49,6 +49,7 @@ jobs:
         uses: actions/checkout@v2
         with:
           repository: ror-community/validation-suite
+          ref: schema-v2
           path: validation-suite
       - name: Validate files
         id: validatefiles
@@ -58,10 +59,11 @@ jobs:
           cd validation-suite
           python -m pip install --upgrade pip
           pip install -r requirements.txt
+          curl https://raw.githubusercontent.com/ror-community/ror-schema/schema-v2/ror_schema_v2_0.json -o ror_schema_v2_0.json
           if [[ -f "../${{github.event.pull_request.head.ref}}/relationships.csv" ]]; then
-              python run_validations.py -i ../files -f ../${{github.event.pull_request.head.ref}}/relationships.csv -p ../files/ --no-geonames
+              python run_validations.py -i ../files -v 2 -s ror_schema_v2_0.json -f ../${{github.event.pull_request.head.ref}}/relationships.csv -p ../files/ --no-geonames
             else
-              python run_validations.py -i ../files --no-geonames
+              python run_validations.py -i ../files -v 2 -s ror_schema_v2_0.json --no-geonames
             fi
       - name: Notify Slack
         if: always()
@@ -71,5 +73,5 @@ jobs:
         with:
           channel: '#ror-curation-releases'
           color: 'good'
-          text: 'Validation status during PR to DEV:  ${{ steps.validatefiles.outcome }}. On branch: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/tree/${env.GITHUB_HEAD_REF}. In directory: ${env.GITHUB_HEAD_REF}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+          text: 'Validation status during PR to DEV:  ${{ steps.validatefiles.outcome }}. Using v2. On branch: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/tree/${env.GITHUB_HEAD_REF}. In directory: ${env.GITHUB_HEAD_REF}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
 

.github/workflows/generate_dump.yml

@@ -2,6 +2,13 @@ name: Create data dump
 on:
   workflow_dispatch:
     inputs:
+      schema-version:
+            required: true
+            description: Release schema version
+            type: choice
+            options:
+            - v1
+            - v2
       new-release:
         type: string
         description: Name of the directory that the new release is located in
@@ -35,7 +42,7 @@ jobs:
         with:
           repository: ror-community/curation_ops
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-          ref: dump-csv
+          ref: v2-crosswalk
           path: ./curation_ops
       - name: Set up Python environment
         uses: actions/setup-python@v2
@@ -45,7 +52,12 @@ jobs:
         id: gendumpscript
         run: |
           cd ./curation_ops/generate_dump/
-          python generate_dump.py -r ${{github.event.inputs.new-release}} -e ${{github.event.inputs.prev-release}} -i '../../ror-records-test' -o '../../ror-records-test'
+          if [[ ${{ github.event.inputs.schema-version }} == 'v1' ]]; then
+            python generate_dump.py -r ${{github.event.inputs.new-release}} -e ${{github.event.inputs.prev-release}} -i '../../ror-records-test' -o '../../ror-records-test' -v 1
+          fi
+          if [[ ${{ github.event.inputs.schema-version }} == 'v2' ]]; then
+            python generate_dump.py -r ${{github.event.inputs.new-release}} -e ${{github.event.inputs.prev-release}} -i '../../ror-records-test' -o '../../ror-records-test' -v 2
+          fi
       - name: cat error file
         if: ${{ steps.gendumpscript.outcome != 'success'}}
         run: |
@@ -73,4 +85,4 @@ jobs:
         with:
           channel: '#ror-curation-releases'
           color: 'good'
-          text: 'DEV Data dump ${{github.event.inputs.new-release}} generation status:  ${{ steps.commitdumpfile.outcome }} Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+          text: 'DEV Data dump ${{github.event.inputs.new-release}} generation status:  ${{ steps.commitdumpfile.outcome }}. Using base version ${{ github.event.inputs.schema-version }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'

.github/workflows/generate_relationships.yml

@@ -2,6 +2,13 @@ name: Create relationships
 on:
   workflow_dispatch:
     inputs:
+      schema-version:
+            required: true
+            description: Schema version
+            type: choice
+            options:
+            - v1
+            - v2
       directory-name:
         type: string
         description: Name of parent directory containing records to generate relationships for. Needed only if different from branch name.
@@ -30,8 +37,13 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install requests==2.23.0
-          curl https://raw.githubusercontent.com/ror-community/curation_ops/main/generate_relationships/generate_relationships.py -o generate_relationships.py
-          python generate_relationships.py relationships.csv
+          curl https://raw.githubusercontent.com/ror-community/curation_ops/v2-crosswalk/generate_relationships/generate_relationships.py -o generate_relationships.py
+          if [[ ${{ github.event.inputs.schema-version }} == 'v1' ]]; then
+            python generate_relationships.py relationships.csv -v 1
+          fi
+          if [[ ${{ github.event.inputs.schema-version }} == 'v2' ]]; then
+            python generate_relationships.py relationships.csv -v 2
+          fi
       - name: commit error file
         if: ${{ steps.genrelshp.outcome != 'success'}}
         working-directory:  ${{ env.WORKING_DIR }}
@@ -42,7 +54,7 @@ jobs:
         if: ${{ steps.genrelshp.outcome == 'success'}}
         uses: stefanzweifel/git-auto-commit-action@v4
         with:
-          file_pattern:  ${{ env.WORKING_DIR }}/**.json
+          file_pattern:  ${{ env.WORKING_DIR }}/*/*.json
           commit_message: Apply relationship changes to files
       - name: Set env variable for notification
         if: always()
@@ -61,4 +73,4 @@ jobs:
         with:
           channel: '#ror-curation-releases'
           color: 'good'
-          text: 'DEV relationship generation status: ${env.VALIDATION_STATUS}. Branch: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/tree/${env.GITHUB_REF_NAME}. Directory: ${{ env.WORKING_DIR }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID} for more details'
+          text: 'DEV relationship generation status: ${env.VALIDATION_STATUS}. Using version ${{ github.event.inputs.schema-version }}. Branch: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/tree/${env.GITHUB_REF_NAME}. Directory: ${{ env.WORKING_DIR }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID} for more details'

.github/workflows/index_dump.py

@@ -28,6 +28,7 @@ def main():
     dataenv = args.dataenv
     headers = args.headers
     full_url = os.path.join(url, filename, dataenv)
+    print(full_url)
     response = send_request(full_url, headers)
     if not(response.ok):
         print(response.text)