Merge pull request #139 from ror-community/index-dump-action

add index dump actions
ror-community · Mar 13, 2024 · cd56d1d · cd56d1d
2 parents b08b694 + 30fbd0e
commit cd56d1d
Show file tree

Hide file tree

Showing 3 changed files with 199 additions and 0 deletions.
diff --git a/.github/workflows/index_dump.py b/.github/workflows/index_dump.py
@@ -0,0 +1,43 @@
+import argparse
+import os
+import requests
+import json
+import sys
+
+def set_args():
+    parser = argparse.ArgumentParser(
+                    description="Send request to index data dump")
+    parser.add_argument('-u', '--url', help='URL to index dump file', required=True)
+    parser.add_argument('-f', '--file', help='name of dump file to be indexed', required=True)
+    parser.add_argument('-e', '--dataenv', help='data env to retrieve dump file from', required=True)
+    parser.add_argument('-he', '--headers', help='key:value json string; headers to authenticate request', required=True, type=json.loads)
+    args = parser.parse_args()
+    return args
+
+def send_request(url, headers):
+    try:
+        response = requests.get(url,headers=headers)
+    except requests.exceptions.RequestException as e:
+        raise e
+    return response
+
+def main():
+    args = set_args()
+    url = args.url
+    filename = args.file
+    dataenv = args.dataenv
+    headers = args.headers
+    full_url = os.path.join(url, filename, dataenv)
+    print(full_url)
+    response = send_request(full_url, headers)
+    if not(response.ok):
+        print(response.text)
+        sys.exit(1)
+    elif response.ok:
+        print(response.text)
+        sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()
+
diff --git a/.github/workflows/prod_index_dump.yml b/.github/workflows/prod_index_dump.yml
@@ -0,0 +1,78 @@
+name: PROD index full data dump
+on:
+  workflow_dispatch:
+    inputs:
+      release-dump:
+        type: string
+        description: Name of existing release dump file to index, without .zip extension (ex v1.41-2024-02-13-ror-data)
+      schema-version:
+        required: true
+        description: Schema version to index
+        default: 'test'
+        type: choice
+        options:
+        - v1
+        - v2
+      data-env:
+        required: true
+        description: ROR data env (test uses ror-data-test repo, prod uses ror-data)
+        default: 'test'
+        type: choice
+        options:
+        - test
+        - prod
+
+
+env:
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
+
+jobs:
+  check_permission:
+    runs-on: ubuntu-latest
+    if: github.event.ref == 'refs/heads/master'
+    steps:
+      - name: Get Permission
+        uses: octokit/[email protected]
+        id: get_permission
+        with:
+          route: GET /repos/${{ github.repository }}/collaborators/${{ github.actor }}/permission
+        env:
+          GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+      - name: output
+        run:  |
+          export role_name=${{ fromJson(steps.get_permission.outputs.data).role_name }}
+          if [[ "$role_name" == "admin" ]]; then
+            exit 0
+          else
+            exit 1
+          fi
+  index-dump:
+    runs-on: ubuntu-latest
+    if: github.event.ref == 'refs/heads/master'
+    needs: check_permission
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: Index dump file
+        id: indexdatadump
+        run: |
+          cd .github/workflows
+          python -m pip install --upgrade pip
+          pip install requests==2.23.0
+          if [[ ${{ github.event.inputs.schema-version }} == "v1" ]]; then
+            python index_dump.py -u ${{ secrets.INDEX_DUMP_PROD_API_URL }} -f ${{ github.event.inputs.release-dump }} -e  ${{ github.event.inputs.data-env }} -he ${{ secrets.INDEX_PROD_API_HEADERS }}
+          else
+            python index_dump.py -u ${{ secrets.INDEX_DUMP_PROD_API_URL_V2 }} -f ${{ github.event.inputs.release-dump }} -e  ${{ github.event.inputs.data-env }} -he ${{ secrets.INDEX_PROD_API_HEADERS }}
+          fi
+      - name: Notify Slack
+        if: always()
+        uses: edge/simple-slack-notify@master
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.CURATOR_SLACK_WEBHOOK_URL }}
+        with:
+          channel: '#ror-curation-releases'
+          color: 'good'
+          text: 'PROD full index from dump status: ${{ steps.indexdatadump.outcome }}. From file: ${{ github.event.inputs.release-dump }}. Schema version: ${{ github.event.inputs.schema-version }}. Data env: ${{ github.event.inputs.data-env }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+
diff --git a/.github/workflows/staging_index_dump.yml b/.github/workflows/staging_index_dump.yml
@@ -0,0 +1,78 @@
+name: STAGING index full data dump
+on:
+  workflow_dispatch:
+    inputs:
+      release-dump:
+        type: string
+        description: Name of existing release dump file to index, without .zip extension (ex v1.41-2024-02-13-ror-data)
+      schema-version:
+        required: true
+        description: Schema version to index
+        default: 'test'
+        type: choice
+        options:
+        - v1
+        - v2
+      data-env:
+        required: true
+        description: ROR data env (test uses ror-data-test repo, prod uses ror-data)
+        default: 'test'
+        type: choice
+        options:
+        - test
+        - prod
+
+
+env:
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
+
+jobs:
+  check_permission:
+    runs-on: ubuntu-latest
+    if: github.event.ref == 'refs/heads/staging'
+    steps:
+      - name: Get Permission
+        uses: octokit/[email protected]
+        id: get_permission
+        with:
+          route: GET /repos/${{ github.repository }}/collaborators/${{ github.actor }}/permission
+        env:
+          GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+      - name: output
+        run:  |
+          export role_name=${{ fromJson(steps.get_permission.outputs.data).role_name }}
+          if [[ "$role_name" == "admin" ]]; then
+            exit 0
+          else
+            exit 1
+          fi
+  index-dump:
+    runs-on: ubuntu-latest
+    if: github.event.ref == 'refs/heads/staging'
+    needs: check_permission
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: Index dump file
+        id: indexdatadump
+        run: |
+          cd .github/workflows
+          python -m pip install --upgrade pip
+          pip install requests==2.23.0
+          if [[ ${{ github.event.inputs.schema-version }} == "v1" ]]; then
+            python index_dump.py -u ${{ secrets.INDEX_DUMP_STAGING_API_URL }} -f ${{ github.event.inputs.release-dump }} -e  ${{ github.event.inputs.data-env }} -he ${{ secrets.INDEX_STAGING_API_HEADERS }}
+          else
+            python index_dump.py -u ${{ secrets.INDEX_DUMP_STAGING_API_URL_V2 }} -f ${{ github.event.inputs.release-dump }} -e  ${{ github.event.inputs.data-env }} -he ${{ secrets.INDEX_STAGING_API_HEADERS }}
+          fi
+      - name: Notify Slack
+        if: always()
+        uses: edge/simple-slack-notify@master
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.CURATOR_SLACK_WEBHOOK_URL }}
+        with:
+          channel: '#ror-curation-releases'
+          color: 'good'
+          text: 'STAGING full index from dump status: ${{ steps.indexdatadump.outcome }}. From file: ${{ github.event.inputs.release-dump }}. Schema version: ${{ github.event.inputs.schema-version }}. Data env: ${{ github.event.inputs.data-env }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+