diff --git a/.github/workflows/index_dump.py b/.github/workflows/index_dump.py new file mode 100644 index 000000000..d52967649 --- /dev/null +++ b/.github/workflows/index_dump.py @@ -0,0 +1,43 @@ +import argparse +import os +import requests +import json +import sys + +def set_args(): + parser = argparse.ArgumentParser( + description="Send request to index data dump") + parser.add_argument('-u', '--url', help='URL to index dump file', required=True) + parser.add_argument('-f', '--file', help='name of dump file to be indexed', required=True) + parser.add_argument('-e', '--dataenv', help='data env to retrieve dump file from', required=True) + parser.add_argument('-he', '--headers', help='key:value json string; headers to authenticate request', required=True, type=json.loads) + args = parser.parse_args() + return args + +def send_request(url, headers): + try: + response = requests.get(url,headers=headers) + except requests.exceptions.RequestException as e: + raise e + return response + +def main(): + args = set_args() + url = args.url + filename = args.file + dataenv = args.dataenv + headers = args.headers + full_url = os.path.join(url, filename, dataenv) + print(full_url) + response = send_request(full_url, headers) + if not(response.ok): + print(response.text) + sys.exit(1) + elif response.ok: + print(response.text) + sys.exit(0) + + +if __name__ == "__main__": + main() + diff --git a/.github/workflows/prod_index_dump.yml b/.github/workflows/prod_index_dump.yml new file mode 100644 index 000000000..008a2e697 --- /dev/null +++ b/.github/workflows/prod_index_dump.yml @@ -0,0 +1,78 @@ +name: PROD index full data dump +on: + workflow_dispatch: + inputs: + release-dump: + type: string + description: Name of existing release dump file to index, without .zip extension (ex v1.41-2024-02-13-ror-data) + schema-version: + required: true + description: Schema version to index + default: 'test' + type: choice + options: + - v1 + - v2 + data-env: + required: true + description: ROR data env (test uses ror-data-test repo, prod uses ror-data) + default: 'test' + type: choice + options: + - test + - prod + + +env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} + +jobs: + check_permission: + runs-on: ubuntu-latest + if: github.event.ref == 'refs/heads/master' + steps: + - name: Get Permission + uses: octokit/request-action@v2.x + id: get_permission + with: + route: GET /repos/${{ github.repository }}/collaborators/${{ github.actor }}/permission + env: + GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + - name: output + run: | + export role_name=${{ fromJson(steps.get_permission.outputs.data).role_name }} + if [[ "$role_name" == "admin" ]]; then + exit 0 + else + exit 1 + fi + index-dump: + runs-on: ubuntu-latest + if: github.event.ref == 'refs/heads/master' + needs: check_permission + steps: + - name: checkout + uses: actions/checkout@v2 + - name: Index dump file + id: indexdatadump + run: | + cd .github/workflows + python -m pip install --upgrade pip + pip install requests==2.23.0 + if [[ ${{ github.event.inputs.schema-version }} == "v1" ]]; then + python index_dump.py -u ${{ secrets.INDEX_DUMP_PROD_API_URL }} -f ${{ github.event.inputs.release-dump }} -e ${{ github.event.inputs.data-env }} -he ${{ secrets.INDEX_PROD_API_HEADERS }} + else + python index_dump.py -u ${{ secrets.INDEX_DUMP_PROD_API_URL_V2 }} -f ${{ github.event.inputs.release-dump }} -e ${{ github.event.inputs.data-env }} -he ${{ secrets.INDEX_PROD_API_HEADERS }} + fi + - name: Notify Slack + if: always() + uses: edge/simple-slack-notify@master + env: + SLACK_WEBHOOK_URL: ${{ secrets.CURATOR_SLACK_WEBHOOK_URL }} + with: + channel: '#ror-curation-releases' + color: 'good' + text: 'PROD full index from dump status: ${{ steps.indexdatadump.outcome }}. From file: ${{ github.event.inputs.release-dump }}. Schema version: ${{ github.event.inputs.schema-version }}. Data env: ${{ github.event.inputs.data-env }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}' + diff --git a/.github/workflows/staging_index_dump.yml b/.github/workflows/staging_index_dump.yml new file mode 100644 index 000000000..8bc08815e --- /dev/null +++ b/.github/workflows/staging_index_dump.yml @@ -0,0 +1,78 @@ +name: STAGING index full data dump +on: + workflow_dispatch: + inputs: + release-dump: + type: string + description: Name of existing release dump file to index, without .zip extension (ex v1.41-2024-02-13-ror-data) + schema-version: + required: true + description: Schema version to index + default: 'test' + type: choice + options: + - v1 + - v2 + data-env: + required: true + description: ROR data env (test uses ror-data-test repo, prod uses ror-data) + default: 'test' + type: choice + options: + - test + - prod + + +env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} + +jobs: + check_permission: + runs-on: ubuntu-latest + if: github.event.ref == 'refs/heads/staging' + steps: + - name: Get Permission + uses: octokit/request-action@v2.x + id: get_permission + with: + route: GET /repos/${{ github.repository }}/collaborators/${{ github.actor }}/permission + env: + GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + - name: output + run: | + export role_name=${{ fromJson(steps.get_permission.outputs.data).role_name }} + if [[ "$role_name" == "admin" ]]; then + exit 0 + else + exit 1 + fi + index-dump: + runs-on: ubuntu-latest + if: github.event.ref == 'refs/heads/staging' + needs: check_permission + steps: + - name: checkout + uses: actions/checkout@v2 + - name: Index dump file + id: indexdatadump + run: | + cd .github/workflows + python -m pip install --upgrade pip + pip install requests==2.23.0 + if [[ ${{ github.event.inputs.schema-version }} == "v1" ]]; then + python index_dump.py -u ${{ secrets.INDEX_DUMP_STAGING_API_URL }} -f ${{ github.event.inputs.release-dump }} -e ${{ github.event.inputs.data-env }} -he ${{ secrets.INDEX_STAGING_API_HEADERS }} + else + python index_dump.py -u ${{ secrets.INDEX_DUMP_STAGING_API_URL_V2 }} -f ${{ github.event.inputs.release-dump }} -e ${{ github.event.inputs.data-env }} -he ${{ secrets.INDEX_STAGING_API_HEADERS }} + fi + - name: Notify Slack + if: always() + uses: edge/simple-slack-notify@master + env: + SLACK_WEBHOOK_URL: ${{ secrets.CURATOR_SLACK_WEBHOOK_URL }} + with: + channel: '#ror-curation-releases' + color: 'good' + text: 'STAGING full index from dump status: ${{ steps.indexdatadump.outcome }}. From file: ${{ github.event.inputs.release-dump }}. Schema version: ${{ github.event.inputs.schema-version }}. Data env: ${{ github.event.inputs.data-env }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}' +