From 699d2ce5c889719cd820b5b9766916e72951a4ff Mon Sep 17 00:00:00 2001
From: lizkrznarich <liz.krznarich@datacite.org>
Date: Tue, 9 Apr 2024 14:25:46 -0500
Subject: [PATCH 1/2] udpate actions for v2

---
 .github/workflows/generate_dump.yml           |  30 ++++-
 .../generate_dump_v2_initial_release.yml      | 106 ++++++++++++++++++
 .github/workflows/main_release.yml            |  12 +-
 .github/workflows/prod_index_dump.yml         |   2 +-
 .github/workflows/prod_manual_index.yml       |  22 +++-
 .github/workflows/staging_manual_index.yml    |  44 ++++++--
 .github/workflows/staging_merge.yml           |  12 +-
 .github/workflows/staging_pull_request.yml    |   8 +-
 .github/workflows/validate.yml                |  71 +++++++++---
 9 files changed, 255 insertions(+), 52 deletions(-)
 create mode 100644 .github/workflows/generate_dump_v2_initial_release.yml

diff --git a/.github/workflows/generate_dump.yml b/.github/workflows/generate_dump.yml
index a1544c522..991074692 100644
--- a/.github/workflows/generate_dump.yml
+++ b/.github/workflows/generate_dump.yml
@@ -2,6 +2,13 @@ name: Create data dump
 on:
   workflow_dispatch:
     inputs:
+      schema-version:
+        required: true
+        description: Release schema version
+        type: choice
+        options:
+        - v1
+        - v2
       new-release:
         type: string
         description: Name of the directory that the new release is located in
@@ -37,6 +44,7 @@ jobs:
         with:
           repository: ror-community/curation_ops
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+          ref: v2-crosswalk
           path: ./curation_ops
       - name: Set up Python environment
         uses: actions/setup-python@v2
@@ -46,7 +54,12 @@ jobs:
         id: gendumpscript
         run: |
           cd ./curation_ops/generate_dump/
-          python generate_dump.py -r ${{github.event.inputs.new-release}} -e ${{github.event.inputs.prev-release}} -i '../../ror-records' -o '../../ror-records'
+          if [[ ${{ github.event.inputs.schema-version }} == 'v1' ]]; then
+            python generate_dump.py -r ${{github.event.inputs.new-release}} -e ${{github.event.inputs.prev-release}} -i '../../ror-records' -o '../../ror-records' -v 1
+          fi
+          if [[ ${{ github.event.inputs.schema-version }} == 'v2' ]]; then
+            python generate_dump.py -r ${{github.event.inputs.new-release}} -e ${{github.event.inputs.prev-release}} -i '../../ror-records' -o '../../ror-records' -v 2
+          fi
       - name: cat error file
         if: ${{ steps.gendumpscript.outcome != 'success'}}
         run: |
@@ -61,11 +74,20 @@ jobs:
         if: ${{ steps.copynewdump.outcome == 'success'}}
         run: |
           cd ./ror-data
-          git config --local user.email "action@github.com"
-          git config --local user.name "GitHub Action"
+          git config --local user.email "ror-bot@ror.org"
+          git config --local user.name "ror-bot"
           git add *.zip
           git commit -m "add new data dump file"
           git push origin main
+      - name: commit changed files
+        if: ${{ steps.commitdumpfile.outcome == 'success'}} && ${{ github.event.inputs.schema-version }} == 'v2'
+        run: |
+          cd ./ror-records
+          git config --local user.email "ror-bot@ror.org"
+          git config --local user.name "ror-bot"
+          git add ${{github.event.inputs.new-release}}/v1/
+          git commit -m "add generated v1 files"
+          git push origin main
       - name: Notify Slack
         if: always()
         uses: edge/simple-slack-notify@master
@@ -74,4 +96,4 @@ jobs:
         with:
           channel: '#ror-curation-releases'
           color: 'good'
-          text: 'PROD Data dump ${{github.event.inputs.new-release}} generation status:  ${{ steps.commitdumpfile.outcome }} Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+          text: 'PROD Data dump ${{github.event.inputs.new-release}} generation status:  ${{ steps.commitdumpfile.outcome }}. Using base version: ${{ github.event.inputs.schema-version }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
diff --git a/.github/workflows/generate_dump_v2_initial_release.yml b/.github/workflows/generate_dump_v2_initial_release.yml
new file mode 100644
index 000000000..ded6fb2cd
--- /dev/null
+++ b/.github/workflows/generate_dump_v2_initial_release.yml
@@ -0,0 +1,106 @@
+name: Create data dump v2 first release
+description: Generates a data dump based on an input file from ror-data-test and uploades the result to ror-data. For use during the first v1 release only.
+on:
+  workflow_dispatch:
+    inputs:
+      schema-version:
+        required: true
+        description: Release schema version
+        type: choice
+        options:
+        - v1
+        - v2
+      new-release:
+        type: string
+        description: Name of the directory that the new release is located in
+      prev-release:
+        type: string
+        description: Name of the existing release zip file to base this data dump from
+
+jobs:
+  generate-dump:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Echo message
+        id: echo_message
+        run: echo "Github action triggered with inputs new release ${{github.event.inputs.new-release}} and previous release ${{github.event.inputs.prev-release}}"
+      - name: checkout ror records repo
+        uses: actions/checkout@v2
+        with:
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+          path: ./ror-records
+      - name: checkout ror data test repo
+        uses: actions/checkout@v2
+        with:
+          repository: ror-community/ror-data-test
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+          path: ./ror-data-test
+      - name: checkout ror data prod repo
+        uses: actions/checkout@v2
+        with:
+          repository: ror-community/ror-data
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+          path: ./ror-data-prod
+      - name: copy previous data dump file
+        id: copyprevdump
+        run: |
+          cp -R ./ror-data-test/${{github.event.inputs.prev-release}}.zip ./ror-records
+      - name: checkout ror curation ops repo
+        uses: actions/checkout@v2
+        with:
+          repository: ror-community/curation_ops
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+          ref: v2-crosswalk
+          path: ./curation_ops
+      - name: Set up Python environment
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.8"
+      - name: execute script
+        id: gendumpscript
+        run: |
+          cd ./curation_ops/generate_dump/
+          if [[ ${{ github.event.inputs.schema-version }} == 'v1' ]]; then
+            python generate_dump.py -r ${{github.event.inputs.new-release}} -e ${{github.event.inputs.prev-release}} -i '../../ror-records' -o '../../ror-records' -v 1
+          fi
+          if [[ ${{ github.event.inputs.schema-version }} == 'v2' ]]; then
+            python generate_dump.py -r ${{github.event.inputs.new-release}} -e ${{github.event.inputs.prev-release}} -i '../../ror-records' -o '../../ror-records' -v 2
+          fi
+      - name: cat error file
+        if: ${{ steps.gendumpscript.outcome != 'success'}}
+        run: |
+          echo "ERRORS found:"
+          cat errors.log
+      - name: copy new data dump file
+        id: copynewdump
+        run: |
+          yes | cp -rf ./ror-records-prod/${{github.event.inputs.new-release}}*.zip ./ror-data-prod
+      - name: commit dump file
+        id: commitdumpfile
+        if: ${{ steps.copynewdump.outcome == 'success'}}
+        run: |
+          cd ./ror-data-prod
+          git config --local user.email "ror-bot@ror.org"
+          git config --local user.name "ror-bot"
+          git add *.zip
+          git commit -m "add new data dump file"
+          git push origin main
+      - name: commit changed files
+        if: ${{ steps.commitdumpfile.outcome == 'success'}} && ${{ github.event.inputs.schema-version }} == 'v2'
+        run: |
+          pwd
+          cd ./ror-records
+          git config --local user.email "ror-bot@ror.org"
+          git config --local user.name "ror-bot"
+          git add ${{github.event.inputs.new-release}}/v1/
+          git commit -m "add generated v1 files"
+          git push origin main
+      - name: Notify Slack
+        if: always()
+        uses: edge/simple-slack-notify@master
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.CURATOR_SLACK_WEBHOOK_URL }}
+        with:
+          channel: '#ror-curation-releases'
+          color: 'good'
+          text: 'DEV Data dump ${{github.event.inputs.new-release}} generation status:  ${{ steps.commitdumpfile.outcome }}. Using base version: ${{ github.event.inputs.schema-version }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
diff --git a/.github/workflows/main_release.yml b/.github/workflows/main_release.yml
index e4039906c..3fd0a2c3f 100644
--- a/.github/workflows/main_release.yml
+++ b/.github/workflows/main_release.yml
@@ -1,4 +1,4 @@
-name: Deploy to Prod
+name: Deploy to Prod v2 only
 on:
   release:
     types: [published]
@@ -50,8 +50,8 @@ jobs:
         run: |
           cd ./${{ github.event.release.tag_name }}
           zip files.zip *.json
-          aws s3 cp files.zip s3://${{ secrets.ROR_DATA_S3_BUCKET_PROD }}/${{ github.event.release.tag_name }}/files.zip
-          aws s3 ls s3://${{ secrets.ROR_DATA_S3_BUCKET_PROD }}/${{ github.event.release.tag_name }}/files.zip
+          aws s3 cp files.zip s3://${{ secrets.ROR_DATA_S3_BUCKET_PROD }}/${{ github.event.release.tag_name }}-v2/files.zip
+          aws s3 ls s3://${{ secrets.ROR_DATA_S3_BUCKET_PROD }}/${{ github.event.release.tag_name }}-v2/files.zip
       - name: Notify Slack
         if: always()
         uses: edge/simple-slack-notify@master
@@ -60,14 +60,14 @@ jobs:
         with:
           channel: '#ror-curation-releases'
           color: 'good'
-          text: 'Copy status to PROD S3 Bucket:  ${{ steps.copyfiles.outcome }}.  From directory: ${{ github.event.release.tag_name }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+          text: 'Copy status to PROD S3 Bucket:  ${{ steps.copyfiles.outcome }}. Using version 2. From directory: ${{ github.event.release.tag_name }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
       - name: Index data
         id: indexdata
         run: |
           cd .github/workflows
           python -m pip install --upgrade pip
           pip install requests==2.23.0
-          python index_files.py -u ${{ secrets.INDEX_PROD_API_URL }} -d ${{ github.event.release.tag_name }} -he ${{ secrets.INDEX_PROD_API_HEADERS }}
+          python index_files.py -u ${{ secrets.INDEX_PROD_API_URL_V2 }} -d ${{ github.event.release.tag_name }}-v2 -he ${{ secrets.INDEX_PROD_API_HEADERS }}
       - name: Notify Slack
         if: always()
         uses: edge/simple-slack-notify@master
@@ -76,4 +76,4 @@ jobs:
         with:
           channel: '#ror-curation-releases'
           color: 'good'
-          text: 'PROD index status: ${{ steps.indexdata.outcome }}. From directory: ${{ github.event.release.tag_name }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+          text: 'PROD index status: ${{ steps.indexdata.outcome }}. Using version 2. From directory: ${{ github.event.release.tag_name }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
diff --git a/.github/workflows/prod_index_dump.yml b/.github/workflows/prod_index_dump.yml
index 3c847efeb..152d39384 100644
--- a/.github/workflows/prod_index_dump.yml
+++ b/.github/workflows/prod_index_dump.yml
@@ -74,5 +74,5 @@ jobs:
         with:
           channel: '#ror-curation-releases'
           color: 'good'
-          text: 'PROD full index from dump status: ${{ steps.indexdatadump.outcome }}. From file: ${{ github.event.inputs.release-dump }}. Schema version: ${{ github.event.inputs.schema-version }}. Data env: ${{ github.event.inputs.data-env }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+          text: 'PROD full index from dump status: ${{ steps.indexdatadump.outcome }}. Using version: ${{ github.event.inputs.schema-version }}. From file: ${{ github.event.inputs.release-dump }}. Schema version: ${{ github.event.inputs.schema-version }}. Data env: ${{ github.event.inputs.data-env }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
 
diff --git a/.github/workflows/prod_manual_index.yml b/.github/workflows/prod_manual_index.yml
index 5afbd3015..68c112261 100644
--- a/.github/workflows/prod_manual_index.yml
+++ b/.github/workflows/prod_manual_index.yml
@@ -2,6 +2,13 @@ name: Manual deploy to Prod
 on:
   workflow_dispatch:
     inputs:
+      schema-version:
+        required: true
+        description: Schema version
+        type: choice
+        options:
+        - v1
+        - v2
       release-tag:
         type: string
         required: true
@@ -49,8 +56,8 @@ jobs:
         run: |
           cd ./${{ github.event.inputs.release-tag }}
           zip files.zip *.json
-          aws s3 cp files.zip s3://${{ secrets.ROR_DATA_S3_BUCKET_PROD }}/${{ github.event.inputs.release-tag }}/files.zip
-          aws s3 ls s3://${{ secrets.ROR_DATA_S3_BUCKET_PROD }}/${{ github.event.inputs.release-tag }}/files.zip
+          aws s3 cp files.zip s3://${{ secrets.ROR_DATA_S3_BUCKET_PROD }}/${{ github.event.inputs.release-tag }}-${{ github.event.inputs.schema-version }}/files.zip
+          aws s3 ls s3://${{ secrets.ROR_DATA_S3_BUCKET_PROD }}/${{ github.event.inputs.release-tag }}-${{ github.event.inputs.schema-version }}/files.zip
       - name: Notify Slack
         if: always()
         uses: edge/simple-slack-notify@master
@@ -59,14 +66,19 @@ jobs:
         with:
           channel: '#ror-curation-releases'
           color: 'good'
-          text: 'Copy status to PROD S3 Bucket:  ${{ steps.copyfiles.outcome }}.  From directory: ${{ github.event.inputs.release-tag }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+          text: 'Copy status to PROD S3 Bucket:  ${{ steps.copyfiles.outcome }}.  Using version: ${{ github.event.inputs.schema-version }}. From directory: ${{ github.event.inputs.release-tag }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
       - name: Index file
         id: indexdata
         run: |
           cd .github/workflows
           python -m pip install --upgrade pip
           pip install requests==2.23.0
-          python index_files.py -u ${{ secrets.INDEX_PROD_API_URL }} -d ${{ github.event.inputs.release-tag }} -he ${{ secrets.INDEX_PROD_API_HEADERS }}
+          if [[ ${{ github.event.inputs.schema-version }} == 'v1' ]]; then
+            python index_files.py -u ${{ secrets.INDEX_PROD_API_URL_V1 }} -d ${{ github.event.inputs.directory-name }}-v1 -he ${{ secrets.INDEX_DEV_API_HEADERS }}
+          fi
+          if [[ ${{ github.event.inputs.schema-version }} == 'v2' ]]; then
+            python index_files.py -u ${{ secrets.INDEX_PROD_API_URL_V2 }} -d ${{ github.event.inputs.directory-name }}-v2 -he ${{ secrets.INDEX_DEV_API_HEADERS }}
+          fi
       - name: Notify Slack
         if: always()
         uses: edge/simple-slack-notify@master
@@ -75,5 +87,5 @@ jobs:
         with:
           channel: '#ror-curation-releases'
           color: 'good'
-          text: 'PROD index status to ES: ${{ steps.indexdata.outcome }}. From directory: ${{ github.event.inputs.release-tag }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+          text: 'PROD index status to ES: ${{ steps.indexdata.outcome }}. Using version:. From directory: ${{ github.event.inputs.release-tag }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
 
diff --git a/.github/workflows/staging_manual_index.yml b/.github/workflows/staging_manual_index.yml
index 0598b49db..c6b422fb2 100644
--- a/.github/workflows/staging_manual_index.yml
+++ b/.github/workflows/staging_manual_index.yml
@@ -2,6 +2,13 @@ name: Manual deploy to Staging
 on:
   workflow_dispatch:
     inputs:
+      schema-version:
+            required: true
+            description: Schema version
+            type: choice
+            options:
+            - v1
+            - v2
       directory-name:
         type: string
         required: true
@@ -49,6 +56,7 @@ jobs:
         uses: actions/checkout@v2
         with:
           repository: ror-community/validation-suite
+          ref: schema-v2
           path: validation-suite
       - name: Validate files
         id: validatefiles
@@ -58,11 +66,22 @@ jobs:
           cd validation-suite
           python -m pip install --upgrade pip
           pip install -r requirements.txt
-          if [[ -f "../${{ github.event.inputs.directory-name }}/relationships.csv" ]]; then
-              python run_validations.py -i ../files -f ../${{ github.event.inputs.directory-name }}/relationships.csv -p ../files/ --no-geonames
-            else
-              python run_validations.py -i ../files --no-geonames
-            fi
+          if [[ ${{ github.event.inputs.schema-version }} == 'v1' ]]; then
+            curl https://raw.githubusercontent.com/ror-community/ror-schema/master/ror_schema.json -o ror_schema.json
+            if [[ -f "../${{github.event.pull_request.head.ref}}/relationships.csv" ]]; then
+                python run_validations.py -i ../files -v 1 -s ror_schema.json -f ../${{github.event.pull_request.head.ref}}/relationships.csv -p ../files/ --no-geonames
+              else
+                python run_validations.py -i ../files -v 1 -s ror_schema.json --no-geonames
+              fi
+          fi
+          if [[ ${{ github.event.inputs.schema-version }} == 'v2' ]]; then
+            curl https://raw.githubusercontent.com/ror-community/ror-schema/master/ror_schema_v2_0.json -o ror_schema_v2_0.json
+            if [[ -f "../${{github.event.pull_request.head.ref}}/relationships.csv" ]]; then
+                python run_validations.py -i ../files -v 2 -s ror_schema_v2_0.json -f ../${{github.event.pull_request.head.ref}}/relationships.csv -p ../files/ --no-geonames
+              else
+                python run_validations.py -i ../files -v 2 -s ror_schema_v2_0.json --no-geonames
+              fi
+          fi
       - name: Notify Slack
         if: always()
         uses: edge/simple-slack-notify@master
@@ -71,7 +90,7 @@ jobs:
         with:
           channel: '#ror-curation-releases'
           color: 'good'
-          text: 'Validation status during STAGING manual index:  ${{ steps.validatefiles.outcome }}. In directory: ${{ github.event.inputs.directory-name }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+          text: 'Validation status during STAGING manual index:  ${{ steps.validatefiles.outcome }}. Using version: ${{ github.event.inputs.schema-version }}. In directory: ${{ github.event.inputs.directory-name }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
   copy-files:
     runs-on: ubuntu-latest
     if: github.event.ref == 'refs/heads/staging'
@@ -89,8 +108,8 @@ jobs:
         run: |
           cd ./${{ github.event.inputs.directory-name }}
           zip files.zip *.json
-          aws s3 cp files.zip s3://${{ secrets.ROR_DATA_S3_BUCKET_STAGING }}/${{ github.event.inputs.directory-name }}/files.zip
-          aws s3 ls s3://${{ secrets.ROR_DATA_S3_BUCKET_STAGING }}/${{ github.event.inputs.directory-name }}/files.zip
+          aws s3 cp files.zip s3://${{ secrets.ROR_DATA_S3_BUCKET_STAGING }}/${{ github.event.inputs.directory-name }}-${{ github.event.inputs.schema-version }}/files.zip
+          aws s3 ls s3://${{ secrets.ROR_DATA_S3_BUCKET_STAGING }}/${{ github.event.inputs.directory-name }}-${{ github.event.inputs.schema-version }}/files.zip
       - name: Notify Slack
         if: always()
         uses: edge/simple-slack-notify@master
@@ -106,7 +125,12 @@ jobs:
           cd .github/workflows
           python -m pip install --upgrade pip
           pip install requests==2.23.0
-          python index_files.py -u ${{ secrets.INDEX_STAGING_API_URL }} -d ${{ github.event.inputs.directory-name }} -he ${{ secrets.INDEX_STAGING_API_HEADERS }}
+          if [[ ${{ github.event.inputs.schema-version }} == 'v1' ]]; then
+            python index_files.py -u ${{ secrets.INDEX_STAGING_API_URLL_V1 }} -d ${{ github.event.inputs.directory-name }}-v1 -he ${{ secrets.INDEX_STAGING_API_HEADERS }}
+          fi
+          if [[ ${{ github.event.inputs.schema-version }} == 'v2' ]]; then
+            python index_files.py -u ${{ secrets.INDEX_STAGING_API_URL_V2 }} -d ${{ github.event.inputs.directory-name }}-v2 -he ${{ secrets.INDEX_STAGING_API_HEADERS }}
+          fi
       - name: Notify Slack
         if: always()
         uses: edge/simple-slack-notify@master
@@ -115,5 +139,5 @@ jobs:
         with:
           channel: '#ror-curation-releases'
           color: 'good'
-          text: 'STAGING index status to ES: ${{ steps.indexdata.outcome }}. From directory: ${{ github.event.inputs.directory-name }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+          text: 'STAGING index status to ES: ${{ steps.indexdata.outcome }}. Using version ${{ github.event.inputs.schema-version }}. From directory: ${{ github.event.inputs.directory-name }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
 
diff --git a/.github/workflows/staging_merge.yml b/.github/workflows/staging_merge.yml
index 66ffc8830..6fa13f37f 100644
--- a/.github/workflows/staging_merge.yml
+++ b/.github/workflows/staging_merge.yml
@@ -1,4 +1,4 @@
-name: Deploy to Staging
+name: Deploy to Staging v2 only
 on:
   push:
     branches:
@@ -39,8 +39,8 @@ jobs:
         run: |
           cd ./${{ env.release_branch }}
           zip files.zip *.json
-          aws s3 cp files.zip s3://${{ secrets.ROR_DATA_S3_BUCKET_STAGING }}/${{ env.release_branch }}/files.zip
-          aws s3 ls s3://${{ secrets.ROR_DATA_S3_BUCKET_STAGING }}/${{ env.release_branch }}/files.zip
+          aws s3 cp files.zip s3://${{ secrets.ROR_DATA_S3_BUCKET_STAGING }}/${{ env.release_branch }}-v2/files.zip
+          aws s3 ls s3://${{ secrets.ROR_DATA_S3_BUCKET_STAGING }}/${{ env.release_branch }}-v2/files.zip
       - name: Notify Slack
         if: always()
         uses: edge/simple-slack-notify@master
@@ -49,14 +49,14 @@ jobs:
         with:
           channel: '#ror-curation-releases'
           color: 'good'
-          text: 'Copy status to staging S3 Bucket:  ${{ steps.copyfiles.outcome }}.  On branch: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/tree/${env.GITHUB_REF_NAME}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+          text: 'Copy status to staging S3 Bucket:  ${{ steps.copyfiles.outcome }}. Using v2. On branch: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/tree/${env.GITHUB_REF_NAME}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
       - name: Index file
         id: indexdata
         run: |
           cd .github/workflows
           python -m pip install --upgrade pip
           pip install requests==2.23.0
-          python index_files.py -u ${{ secrets.INDEX_STAGING_API_URL }} -d ${{ env.release_branch }} -he ${{ secrets.INDEX_STAGING_API_HEADERS }}
+          python index_files.py -u ${{ secrets.INDEX_STAGING_API_URL_V2 }} -d ${{ env.release_branch }}-v2 -he ${{ secrets.INDEX_STAGING_API_HEADERS }}
       - name: Notify Slack
         if: always()
         uses: edge/simple-slack-notify@master
@@ -65,5 +65,5 @@ jobs:
         with:
           channel: '#ror-curation-releases'
           color: 'good'
-          text: 'Staging index status: ${{ steps.indexdata.outcome }}. On branch: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/tree/${env.GITHUB_REF_NAME}. Directory: ${{ env.release_branch }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+          text: 'Staging index status: ${{ steps.indexdata.outcome }}. Using v2. On branch: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/tree/${env.GITHUB_REF_NAME}. Directory: ${{ env.release_branch }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
 
diff --git a/.github/workflows/staging_pull_request.yml b/.github/workflows/staging_pull_request.yml
index 940f87735..116e1743f 100644
--- a/.github/workflows/staging_pull_request.yml
+++ b/.github/workflows/staging_pull_request.yml
@@ -49,6 +49,7 @@ jobs:
         uses: actions/checkout@v2
         with:
           repository: ror-community/validation-suite
+          ref: schema-v2
           path: validation-suite
       - name: Validate files
         id: validatefiles
@@ -58,10 +59,11 @@ jobs:
           cd validation-suite
           python -m pip install --upgrade pip
           pip install -r requirements.txt
+          curl https://raw.githubusercontent.com/ror-community/ror-schema/master/ror_schema_v2_0.json -o ror_schema_v2_0.json
           if [[ -f "../${{github.event.pull_request.head.ref}}/relationships.csv" ]]; then
-              python run_validations.py -i ../files -f ../${{github.event.pull_request.head.ref}}/relationships.csv -p ../files/ --no-geonames
+              python run_validations.py -i ../files -v 2 -s ror_schema_v2_0.json -f ../${{github.event.pull_request.head.ref}}/relationships.csv -p ../files/ --no-geonames
             else
-              python run_validations.py -i ../files --no-geonames
+              python run_validations.py -i ../files -v 2 -s ror_schema_v2_0.json --no-geonames
             fi
       - name: Notify Slack
         if: always()
@@ -71,5 +73,5 @@ jobs:
         with:
           channel: '#ror-curation-releases'
           color: 'good'
-          text: 'Validation status during PR to STAGING:  ${{ steps.validatefiles.outcome }}. On branch: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/tree/${env.GITHUB_HEAD_REF}. In directory: ${env.GITHUB_HEAD_REF}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+          text: 'Validation status during PR to STAGING:  ${{ steps.validatefiles.outcome }}. Using v2. On branch: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/tree/${env.GITHUB_HEAD_REF}. In directory: ${env.GITHUB_HEAD_REF}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
 
diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml
index 92cef6644..c0adc3897 100644
--- a/.github/workflows/validate.yml
+++ b/.github/workflows/validate.yml
@@ -2,6 +2,13 @@ name: Validate files
 on:
   workflow_dispatch:
     inputs:
+      schema-version:
+        required: true
+        description: Schema version
+        type: choice
+        options:
+        - v1
+        - v2
       with-relationship:
         type: boolean
         description: Check box to validate with relationships
@@ -29,6 +36,7 @@ jobs:
         uses: actions/checkout@v2
         with:
           repository: ror-community/validation-suite
+          ref: schema-v2
           path: validation-suite
       - name: Get directory name
         if: "${{ github.event.inputs.directory-name != '' }}"
@@ -44,26 +52,55 @@ jobs:
           cd validation-suite
           python -m pip install --upgrade pip
           pip install -r requirements.txt
+          curl https://raw.githubusercontent.com/ror-community/ror-schema/master/ror_schema.json -o ror_schema.json
+          curl https://raw.githubusercontent.com/ror-community/ror-schema/master/ror_schema_v2_0.json -o ror_schema_v2_0.json
           ls ../files/
-          if [[ ${{ github.event.inputs.skip-geonames }} == true ]]; then
-            if [[ ${{ github.event.inputs.with-relationship }} == true ]]; then
-              if [[ -f "../${{ env.WORKING_DIR }}/relationships.csv" ]]; then
-                python run_validations.py -i ../files -f ../${{ env.WORKING_DIR }}/relationships.csv -p ../files/ --no-geonames
-              else
-                python run_validations.py -i ../files -p ../files/ --no-geonames
+          if [[ ${{ github.event.inputs.schema-version }} == 'v1' ]]; then
+            echo "validating v1"
+            if [[ ${{ github.event.inputs.skip-geonames }} == true ]]; then
+              if [[ ${{ github.event.inputs.with-relationship }} == true ]]; then
+                if [[ -f "../${{ env.WORKING_DIR }}/relationships.csv" ]]; then
+                  python run_validations.py -i ../files -v 1 -s ror_schema.json -f ../${{ env.WORKING_DIR }}/relationships.csv -p ../files/ --no-geonames
+                else
+                  python run_validations.py -i ../files -v 1 -s ror_schema.json -p ../files/ --no-geonames
+                fi
+              elif [[ ${{ github.event.inputs.with-relationship }} == false ]]; then
+                python run_validations.py -i ../files -v 1 -s ror_schema.json --no-geonames
+              fi
+            else
+              if [[ ${{ github.event.inputs.with-relationship }} == true ]]; then
+                if [[ -f "../${{ env.WORKING_DIR }}/relationships.csv" ]]; then
+                  python run_validations.py -i ../files -v 1 -s ror_schema.json -f ../${{ env.WORKING_DIR }}/relationships.csv -p ../files/
+                else
+                  python run_validations.py -i ../files -v 1 -s ror_schema.json -p ../files/
+                fi
+              elif [[ ${{ github.event.inputs.with-relationship }} == false ]]; then
+                python run_validations.py -i ../files -v 1 -s ror_schema.json
               fi
-            elif [[ ${{ github.event.inputs.with-relationship }} == false ]]; then
-              python run_validations.py -i ../files --no-geonames
             fi
-          else
-            if [[ ${{ github.event.inputs.with-relationship }} == true ]]; then
-              if [[ -f "../${{ env.WORKING_DIR }}/relationships.csv" ]]; then
-                python run_validations.py -i ../files -f ../${{ env.WORKING_DIR }}/relationships.csv -p ../files/
-              else
-                python run_validations.py -i ../files -p ../files/
+          fi
+          if [[ ${{ github.event.inputs.schema-version }} == 'v2' ]]; then
+            echo "validating v2"
+            if [[ ${{ github.event.inputs.skip-geonames }} == true ]]; then
+              if [[ ${{ github.event.inputs.with-relationship }} == true ]]; then
+                if [[ -f "../${{ env.WORKING_DIR }}/relationships.csv" ]]; then
+                  python run_validations.py -i ../files -v 2 -s ror_schema_v2_0.json -f ../${{ env.WORKING_DIR }}/relationships.csv -p ../files/ --no-geonames
+                else
+                  python run_validations.py -i ../files -v 2 -s ror_schema_v2_0.json -p ../files/ --no-geonames
+                fi
+              elif [[ ${{ github.event.inputs.with-relationship }} == false ]]; then
+                python run_validations.py -i ../files -v 2 -s ror_schema_v2_0.json --no-geonames
+              fi
+            else
+              if [[ ${{ github.event.inputs.with-relationship }} == true ]]; then
+                if [[ -f "../${{ env.WORKING_DIR }}/relationships.csv" ]]; then
+                  python run_validations.py -i ../files -v 2 -s ror_schema_v2_0.json -f ../${{ env.WORKING_DIR }}/relationships.csv -p ../files/
+                else
+                  python run_validations.py -i ../files -v 2 -s ror_schema_v2_0.json -p ../files/
+                fi
+              elif [[ ${{ github.event.inputs.with-relationship }} == false ]]; then
+                python run_validations.py -i ../files -v 2 -s ror_schema_v2_0.json
               fi
-            elif [[ ${{ github.event.inputs.with-relationship }} == false ]]; then
-              python run_validations.py -i ../files
             fi
           fi
       - name: Notify Slack
@@ -74,4 +111,4 @@ jobs:
         with:
           channel: '#ror-curation-releases'
           color: 'good'
-          text: 'Validation status ${{ steps.validatefiles.outcome }} in ${env.GITHUB_REPOSITORY}. On branch: ${{ env.WORKING_DIR }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+          text: 'Validation status ${{ steps.validatefiles.outcome }} in ${env.GITHUB_REPOSITORY}. On branch: ${{ env.WORKING_DIR }}. Using version ${{ github.event.inputs.schema-version }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'

From 652f5551051c9abb7bcfae29641cfb61087c7fc3 Mon Sep 17 00:00:00 2001
From: lizkrznarich <liz.krznarich@datacite.org>
Date: Wed, 10 Apr 2024 18:14:06 -0500
Subject: [PATCH 2/2] remove special dump action and temp update generate dump
 to use ror-data-test

---
 .github/workflows/generate_dump.yml           |   2 +-
 .../generate_dump_v2_initial_release.yml      | 106 ------------------
 2 files changed, 1 insertion(+), 107 deletions(-)
 delete mode 100644 .github/workflows/generate_dump_v2_initial_release.yml

diff --git a/.github/workflows/generate_dump.yml b/.github/workflows/generate_dump.yml
index 991074692..c945fc400 100644
--- a/.github/workflows/generate_dump.yml
+++ b/.github/workflows/generate_dump.yml
@@ -32,7 +32,7 @@ jobs:
       - name: checkout ror data repo
         uses: actions/checkout@v2
         with:
-          repository: ror-community/ror-data
+          repository: ror-community/ror-data-test
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
           path: ./ror-data
       - name: copy previous data dump file
diff --git a/.github/workflows/generate_dump_v2_initial_release.yml b/.github/workflows/generate_dump_v2_initial_release.yml
deleted file mode 100644
index ded6fb2cd..000000000
--- a/.github/workflows/generate_dump_v2_initial_release.yml
+++ /dev/null
@@ -1,106 +0,0 @@
-name: Create data dump v2 first release
-description: Generates a data dump based on an input file from ror-data-test and uploades the result to ror-data. For use during the first v1 release only.
-on:
-  workflow_dispatch:
-    inputs:
-      schema-version:
-        required: true
-        description: Release schema version
-        type: choice
-        options:
-        - v1
-        - v2
-      new-release:
-        type: string
-        description: Name of the directory that the new release is located in
-      prev-release:
-        type: string
-        description: Name of the existing release zip file to base this data dump from
-
-jobs:
-  generate-dump:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Echo message
-        id: echo_message
-        run: echo "Github action triggered with inputs new release ${{github.event.inputs.new-release}} and previous release ${{github.event.inputs.prev-release}}"
-      - name: checkout ror records repo
-        uses: actions/checkout@v2
-        with:
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-          path: ./ror-records
-      - name: checkout ror data test repo
-        uses: actions/checkout@v2
-        with:
-          repository: ror-community/ror-data-test
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-          path: ./ror-data-test
-      - name: checkout ror data prod repo
-        uses: actions/checkout@v2
-        with:
-          repository: ror-community/ror-data
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-          path: ./ror-data-prod
-      - name: copy previous data dump file
-        id: copyprevdump
-        run: |
-          cp -R ./ror-data-test/${{github.event.inputs.prev-release}}.zip ./ror-records
-      - name: checkout ror curation ops repo
-        uses: actions/checkout@v2
-        with:
-          repository: ror-community/curation_ops
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-          ref: v2-crosswalk
-          path: ./curation_ops
-      - name: Set up Python environment
-        uses: actions/setup-python@v2
-        with:
-          python-version: "3.8"
-      - name: execute script
-        id: gendumpscript
-        run: |
-          cd ./curation_ops/generate_dump/
-          if [[ ${{ github.event.inputs.schema-version }} == 'v1' ]]; then
-            python generate_dump.py -r ${{github.event.inputs.new-release}} -e ${{github.event.inputs.prev-release}} -i '../../ror-records' -o '../../ror-records' -v 1
-          fi
-          if [[ ${{ github.event.inputs.schema-version }} == 'v2' ]]; then
-            python generate_dump.py -r ${{github.event.inputs.new-release}} -e ${{github.event.inputs.prev-release}} -i '../../ror-records' -o '../../ror-records' -v 2
-          fi
-      - name: cat error file
-        if: ${{ steps.gendumpscript.outcome != 'success'}}
-        run: |
-          echo "ERRORS found:"
-          cat errors.log
-      - name: copy new data dump file
-        id: copynewdump
-        run: |
-          yes | cp -rf ./ror-records-prod/${{github.event.inputs.new-release}}*.zip ./ror-data-prod
-      - name: commit dump file
-        id: commitdumpfile
-        if: ${{ steps.copynewdump.outcome == 'success'}}
-        run: |
-          cd ./ror-data-prod
-          git config --local user.email "ror-bot@ror.org"
-          git config --local user.name "ror-bot"
-          git add *.zip
-          git commit -m "add new data dump file"
-          git push origin main
-      - name: commit changed files
-        if: ${{ steps.commitdumpfile.outcome == 'success'}} && ${{ github.event.inputs.schema-version }} == 'v2'
-        run: |
-          pwd
-          cd ./ror-records
-          git config --local user.email "ror-bot@ror.org"
-          git config --local user.name "ror-bot"
-          git add ${{github.event.inputs.new-release}}/v1/
-          git commit -m "add generated v1 files"
-          git push origin main
-      - name: Notify Slack
-        if: always()
-        uses: edge/simple-slack-notify@master
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.CURATOR_SLACK_WEBHOOK_URL }}
-        with:
-          channel: '#ror-curation-releases'
-          color: 'good'
-          text: 'DEV Data dump ${{github.event.inputs.new-release}} generation status:  ${{ steps.commitdumpfile.outcome }}. Using base version: ${{ github.event.inputs.schema-version }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'