Skip to content
Routine submodule updates

testing variable geneartion #422

Sign in to view logs

testing variable geneartion

testing variable geneartion #422

Workflow file for this run

.github/workflows/main.yml at fc2cd9a
name: Routine submodule updates
on:
schedule:
- cron: '0 9 * * *' # once a day at 9 am runs
push:
branches:
- main
pull_request:
branches:
- main
jobs:
housekeeping:
runs-on: ubuntu-latest
steps:
- name: Checkout repository and submodules
uses: actions/checkout@master
- name: Checkout latest submodules
run: git submodule update --init --recursive --remote
- name: List files in the repository (Useful for debugging)
run: |
tree ${{ github.workspace }}
- name: setup python
uses: actions/setup-python@v4
with:
python-version: 3.10.6 #install the python needed
- name: Install requirements
run: |
pip install -r ${{ github.workspace }}/code/sdc.tests/code/requirements.txt
- name: Run all tests (that match with test_*.py under the sdc.tests submodule)
continue-on-error: true
run: |
for f in code/sdc.tests/code/tests/test_*.py; do python "$f"; done
- name: Run data inventory
run: |
python code/inventory/data_inventory.py
- name: Combine all measure_info into measure_info_all (this is the current state, not the known correct state)
run: |
python code/combine_measure_info.py
- name: Run geographies list
run: |
python code/inventory/geographies_list.py
- name: Adding index.html with reference to all the sites
run: |
rm docs/index.html
printf "<h3>Data Commons: Test and Inventory Results</h3>" >> docs/index.html
find docs -name "*.html" | cut -c 5- > docs/temp
printf "<h4>Code Tests</h4>" >> docs/index.html
printf "<ol>\n" >> docs/index.html
cat docs/temp | while read line; do if echo ${line} | grep "test_code"; then printf "\t<li><a href='https://uva-bi-sdad.github.io/sdc.all_dev/${line}'> $line </a></li>\n" >> docs/index.html; fi; done
printf "</ol>\n" >> docs/index.html
printf "<h4>Data Tests</h4>" >> docs/index.html
printf "<ol>\n" >> docs/index.html
cat docs/temp | while read line; do if echo ${line} | grep "test_data"; then printf "\t<li><a href='https://uva-bi-sdad.github.io/sdc.all_dev/${line}'> $line </a></li>\n" >> docs/index.html; fi; done
printf "</ol>\n" >> docs/index.html
printf "<h4>File Name Tests</h4>" >> docs/index.html
printf "<ol>\n" >> docs/index.html
cat docs/temp | while read line; do if echo ${line} | grep "test_file_name"; then printf "\t<li><a href='https://uva-bi-sdad.github.io/sdc.all_dev/${line}'> $line </a></li>\n" >> docs/index.html; fi; done
printf "</ol>\n" >> docs/index.html
printf "<h4>Measure Info Tests</h4>" >> docs/index.html
printf "<ol>\n" >> docs/index.html
cat docs/temp | while read line; do if echo ${line} | grep "test_measure_info"; then printf "\t<li><a href='https://uva-bi-sdad.github.io/sdc.all_dev/${line}'> $line </a></li>\n" >> docs/index.html; fi; done
printf "</ol>\n" >> docs/index.html
printf "<h4>Inventory Reports</h4>" >> docs/index.html
printf "<ol>\n" >> docs/index.html
cat docs/temp | while read line; do if echo ${line} | grep "inventory"; then printf "\t<li><a href='https://uva-bi-sdad.github.io/sdc.all_dev/${line}'> $line </a></li>\n" >> docs/index.html; fi; done
printf "</ol>\n" >> docs/index.html
rm docs/temp
- name: Commit the results
run: |
git config --local user.email "[email protected]"
git config --local user.name "GitHub Actions"
git add .
git commit -m "Automated manifest update"
- name: Push changes to current repository
uses: ad-m/github-push-action@master
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
branch: ${{ github.ref }}
- name: Remove github things # Required so that you can push to another directory
run: |
rm -rf .git
- name: Remove github submodules # Required so that you can push to another directory
run: |
rm .gitmodules
- name: Remove github workflow # Required so that you can push to another directory
run: |
rm -rf .github
- name: Push directory to another repository
uses: cpina/github-action-push-to-another-repository@main
env:
SSH_DEPLOY_KEY: ${{ secrets.SSH_DEPLOY_KEY }}
with:
source-directory: .
destination-github-username: 'uva-bi-sdad'
destination-repository-name: 'sdc.all'
target-branch: main
- name: Generate a variables folder and export all the data there
run: mkdir -p variables
- name: Check out editor repository code
uses: actions/checkout@master
with:
repository: uva-bi-sdad/sdc.editor
path: sdc.editor/
- name: Install requirements for the editor
run: |
pip install -r sdc.editor/code/requirements.txt
- name: Generate the variables data
run: |
python sdc.editor/code/edit_99_combine_split_by_variable.py -i ${{ github.repository }} -o ${{ github.repository }}/variables
- name: Push directory to another repository
uses: cpina/github-action-push-to-another-repository@main
env:
SSH_DEPLOY_KEY: ${{ secrets.VARIABLES_DEPLOY_KEY }}
with:
source-directory: .
destination-github-username: 'uva-bi-sdad'
destination-repository-name: 'sdc.variables'
target-branch: main
on_fail:
if: ${{ failure() }} # still push the total public one to sdc.all on fail
needs: housekeeping
runs-on: ubuntu-latest
steps:
- name: Checkout repository and submodules
uses: actions/checkout@master
- name: Checkout latest submodules
run: git submodule update --init --recursive --remote
- name: Remove github things # Required so that you can push to another directory
run: |
rm -rf .git
- name: Remove github submodules # Required so that you can push to another directory
run: |
rm .gitmodules
- name: Remove github workflow # Required so that you can push to another directory
run: |
rm -rf .github
- name: Push directory to another repository
uses: cpina/github-action-push-to-another-repository@main
env:
SSH_DEPLOY_KEY: ${{ secrets.SSH_DEPLOY_KEY }}
with:
source-directory: .
destination-github-username: 'uva-bi-sdad'
destination-repository-name: 'sdc.all'
target-branch: main