Skip to content

Data Pipeline

Data Pipeline #551

Workflow file for this run

name: Data Pipeline
"on":
workflow_dispatch:
schedule:
# * is a special character in YAML so you have to quote this string
- cron: "30 23 * * *"
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: "3.10"
cache: pipenv
- name: Install pipenv
run: curl https://raw.githubusercontent.com/pypa/pipenv/master/get-pipenv.py | python
- name: Install dependencies
run: pipenv sync
- name: Setup credentials file
env:
OI_SFTP_USERNAME: ${{ secrets.OI_SFTP_USERNAME }}
OI_SFTP_PASSWORD: ${{ secrets.OI_SFTP_PASSWORD }}
run: |
mkdir credentials
echo ${{ secrets.FIREBASE_SERVICE_ACCOUNT_KEY }} | base64 -d > ./credentials/serviceAccountKey.json
pipenv run dvc remote modify --local oicloud user ${{ secrets.OI_SFTP_USERNAME }}
pipenv run dvc remote modify --local oicloud password ${{ secrets.OI_SFTP_PASSWORD }}
- name: Run DVC pipelines
env:
PYTHONPATH: scripts
GOOGLE_APPLICATION_CREDENTIALS: "./credentials/serviceAccountKey.json"
AIRTABLE_API_KEY: ${{ secrets.AIRTABLE_API_KEY }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
TYPEFORM_PRIVATE_TOKEN: ${{ secrets.TYPEFORM_PRIVATE_TOKEN }}
TICKET_TAILOR_API_KEY: ${{ secrets.TICKET_TAILOR_API_KEY }}
OI_SFTP_HOST: ${{ secrets.OI_SFTP_HOST }}
OI_SFTP_USERNAME: ${{ secrets.OI_SFTP_USERNAME }}
OI_SFTP_PASSWORD: ${{ secrets.OI_SFTP_PASSWORD }}
run: |
# dvc pull
pipenv run dvc update -R working
pipenv run dvc repro -R scripts
- name: Commit and Push changes
uses: stefanzweifel/git-auto-commit-action@v4
with:
commit_message: Data updates
file_pattern: data docs working