Routine submodule updates #733
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Routine submodule updates | |
| on: | |
| schedule: | |
| - cron: '0 21 * * 1-5' # utc | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - main | |
| # Cacnel previous runs if a new push is sent | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| housekeeping: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: write | |
| pages: write | |
| steps: | |
| - name: Checkout repository and submodules | |
| uses: actions/checkout@master | |
| - name: Checkout latest submodules | |
| run: git submodule update --init --recursive --remote | |
| - name: List files in the repository (Useful for debugging) | |
| run: | | |
| tree ${{ github.workspace }} | |
| - name: setup python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.10.6 #install the python needed | |
| - name: Install requirements | |
| run: | | |
| pip install -r ${{ github.workspace }}/code/sdc.tests/code/requirements.txt | |
| - name: Run all tests (that match with test_*.py under the sdc.tests submodule) | |
| continue-on-error: true | |
| run: | | |
| for f in code/sdc.tests/code/tests/test_*.py; do python "$f"; done | |
| - name: Run data inventory | |
| run: | | |
| python code/inventory/data_inventory.py | |
| - name: Combine all measure_info into measure_info_all (this is the current state, not the known correct state) | |
| run: | | |
| python code/combine_measure_info.py | |
| - name: Run geographies list | |
| run: | | |
| python code/inventory/geographies_list.py | |
| - name: Adding index.html with reference to all the sites | |
| run: | | |
| rm docs/index.html | |
| printf "<h3>Data Commons: Test and Inventory Results</h3>" >> docs/index.html | |
| find docs -name "*.html" | cut -c 5- > docs/temp | |
| printf "<h4>Code Tests</h4>" >> docs/index.html | |
| printf "<ol>\n" >> docs/index.html | |
| cat docs/temp | while read line; do if echo ${line} | grep "test_code"; then printf "\t<li><a href='https://uva-bi-sdad.github.io/sdc.all_dev/${line}'> $line </a></li>\n" >> docs/index.html; fi; done | |
| printf "</ol>\n" >> docs/index.html | |
| printf "<h4>Data Tests</h4>" >> docs/index.html | |
| printf "<ol>\n" >> docs/index.html | |
| cat docs/temp | while read line; do if echo ${line} | grep "test_data"; then printf "\t<li><a href='https://uva-bi-sdad.github.io/sdc.all_dev/${line}'> $line </a></li>\n" >> docs/index.html; fi; done | |
| printf "</ol>\n" >> docs/index.html | |
| printf "<h4>File Name Tests</h4>" >> docs/index.html | |
| printf "<ol>\n" >> docs/index.html | |
| cat docs/temp | while read line; do if echo ${line} | grep "test_file_name"; then printf "\t<li><a href='https://uva-bi-sdad.github.io/sdc.all_dev/${line}'> $line </a></li>\n" >> docs/index.html; fi; done | |
| printf "</ol>\n" >> docs/index.html | |
| printf "<h4>Measure Info Tests</h4>" >> docs/index.html | |
| printf "<ol>\n" >> docs/index.html | |
| cat docs/temp | while read line; do if echo ${line} | grep "test_measure_info"; then printf "\t<li><a href='https://uva-bi-sdad.github.io/sdc.all_dev/${line}'> $line </a></li>\n" >> docs/index.html; fi; done | |
| printf "</ol>\n" >> docs/index.html | |
| printf "<h4>Inventory Reports</h4>" >> docs/index.html | |
| printf "<ol>\n" >> docs/index.html | |
| cat docs/temp | while read line; do if echo ${line} | grep "inventory"; then printf "\t<li><a href='https://uva-bi-sdad.github.io/sdc.all_dev/${line}'> $line </a></li>\n" >> docs/index.html; fi; done | |
| printf "</ol>\n" >> docs/index.html | |
| rm docs/temp | |
| - name: Commit the results | |
| run: | | |
| git config --local user.email "[email protected]" | |
| git config --local user.name "GitHub Actions" | |
| git add . | |
| git commit -m "Automated manifest update" | |
| - name: Push changes to current repository | |
| uses: ad-m/github-push-action@master | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| branch: ${{ github.ref }} | |
| - name: Remove github things # Required so that you can push to another directory | |
| run: | | |
| rm -rf .git | |
| - name: Remove github submodules # Required so that you can push to another directory | |
| run: | | |
| rm .gitmodules | |
| - name: Remove github workflow # Required so that you can push to another directory | |
| run: | | |
| rm -rf .github | |
| - name: Push directory to another repository | |
| uses: cpina/github-action-push-to-another-repository@main | |
| env: | |
| SSH_DEPLOY_KEY: ${{ secrets.SSH_DEPLOY_KEY }} | |
| with: | |
| source-directory: . | |
| destination-github-username: 'uva-bi-sdad' | |
| destination-repository-name: 'sdc.all' | |
| target-branch: main | |
| # Running the GitHub pages here so that you don't need to run runners twice | |
| - name: Setup Pages | |
| uses: actions/configure-pages@v3 | |
| - name: Upload artifact | |
| uses: actions/upload-pages-artifact@v2 | |
| with: | |
| path: '${{ github.workspace }}/docs' | |
| - name: Deploy to GitHub Pages | |
| id: deployment | |
| uses: actions/deploy-pages@v2 | |
| - name: Generate a measures folder and export all the data there | |
| run: mkdir -p measures | |
| - name: Check out editor repository code | |
| uses: actions/checkout@master | |
| with: | |
| repository: uva-bi-sdad/sdc.editor | |
| path: ${{ github.workspace }}/sdc.editor | |
| - name: List files in the repository (Useful for debugging) | |
| run: | | |
| ls -la ${{ github.workspace }}/sdc.editor | |
| - name: Install requirements for the editor | |
| run: | | |
| pip install -r ${{ github.workspace }}/sdc.editor/code/requirements.txt | |
| - name: Copy over a measure_info_all.json file | |
| run: | | |
| cp ${{ github.workspace }}/data/measure_info_all.json ${{ github.workspace }}/measures | |
| - name: Generate the measures data | |
| run: | | |
| python ${{ github.workspace }}/sdc.editor/code/sdc.all_edit_combine_split_by_variable.py -i ${{ github.workspace }} -o ${{ github.workspace }}/measures | |
| - name: Push directory to another repository | |
| uses: cpina/github-action-push-to-another-repository@main | |
| env: | |
| SSH_DEPLOY_KEY: ${{ secrets.VARIABLES_DEPLOY_KEY }} | |
| with: | |
| source-directory: measures | |
| destination-github-username: 'uva-bi-sdad' | |
| destination-repository-name: 'sdc.measures' | |
| target-branch: main | |
| on_fail: | |
| if: ${{ failure() }} # still push the total public one to sdc.all on fail | |
| needs: housekeeping | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository and submodules | |
| uses: actions/checkout@master | |
| - name: Checkout latest submodules | |
| run: git submodule update --init --recursive --remote | |
| - name: Remove github things # Required so that you can push to another directory | |
| run: | | |
| rm -rf .git | |
| - name: Remove github submodules # Required so that you can push to another directory | |
| run: | | |
| rm .gitmodules | |
| - name: Remove github workflow # Required so that you can push to another directory | |
| run: | | |
| rm -rf .github | |
| - name: Push directory to another repository | |
| uses: cpina/github-action-push-to-another-repository@main | |
| env: | |
| SSH_DEPLOY_KEY: ${{ secrets.SSH_DEPLOY_KEY }} | |
| with: | |
| source-directory: . | |
| destination-github-username: 'uva-bi-sdad' | |
| destination-repository-name: 'sdc.all' | |
| target-branch: main |