-
Notifications
You must be signed in to change notification settings - Fork 0
169 lines (166 loc) · 7.25 KB
/
main.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
name: Routine submodule updates
on:
schedule:
- cron: '0 9 * * *' # once a day at 9 am runs
push:
branches:
- main
pull_request:
branches:
- main
# Cacnel previous runs if a new push is sent
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
housekeeping:
runs-on: ubuntu-latest
permissions:
id-token: write
contents: write
pages: write
steps:
- name: Checkout repository and submodules
uses: actions/checkout@master
- name: Checkout latest submodules
run: git submodule update --init --recursive --remote
- name: List files in the repository (Useful for debugging)
run: |
tree ${{ github.workspace }}
- name: setup python
uses: actions/setup-python@v4
with:
python-version: 3.10.6 #install the python needed
- name: Install requirements
run: |
pip install -r ${{ github.workspace }}/code/sdc.tests/code/requirements.txt
- name: Run all tests (that match with test_*.py under the sdc.tests submodule)
continue-on-error: true
run: |
for f in code/sdc.tests/code/tests/test_*.py; do python "$f"; done
- name: Run data inventory
run: |
python code/inventory/data_inventory.py
- name: Combine all measure_info into measure_info_all (this is the current state, not the known correct state)
run: |
python code/combine_measure_info.py
- name: Run geographies list
run: |
python code/inventory/geographies_list.py
- name: Adding index.html with reference to all the sites
run: |
rm docs/index.html
printf "<h3>Data Commons: Test and Inventory Results</h3>" >> docs/index.html
find docs -name "*.html" | cut -c 5- > docs/temp
printf "<h4>Code Tests</h4>" >> docs/index.html
printf "<ol>\n" >> docs/index.html
cat docs/temp | while read line; do if echo ${line} | grep "test_code"; then printf "\t<li><a href='https://uva-bi-sdad.github.io/sdc.all_dev/${line}'> $line </a></li>\n" >> docs/index.html; fi; done
printf "</ol>\n" >> docs/index.html
printf "<h4>Data Tests</h4>" >> docs/index.html
printf "<ol>\n" >> docs/index.html
cat docs/temp | while read line; do if echo ${line} | grep "test_data"; then printf "\t<li><a href='https://uva-bi-sdad.github.io/sdc.all_dev/${line}'> $line </a></li>\n" >> docs/index.html; fi; done
printf "</ol>\n" >> docs/index.html
printf "<h4>File Name Tests</h4>" >> docs/index.html
printf "<ol>\n" >> docs/index.html
cat docs/temp | while read line; do if echo ${line} | grep "test_file_name"; then printf "\t<li><a href='https://uva-bi-sdad.github.io/sdc.all_dev/${line}'> $line </a></li>\n" >> docs/index.html; fi; done
printf "</ol>\n" >> docs/index.html
printf "<h4>Measure Info Tests</h4>" >> docs/index.html
printf "<ol>\n" >> docs/index.html
cat docs/temp | while read line; do if echo ${line} | grep "test_measure_info"; then printf "\t<li><a href='https://uva-bi-sdad.github.io/sdc.all_dev/${line}'> $line </a></li>\n" >> docs/index.html; fi; done
printf "</ol>\n" >> docs/index.html
printf "<h4>Inventory Reports</h4>" >> docs/index.html
printf "<ol>\n" >> docs/index.html
cat docs/temp | while read line; do if echo ${line} | grep "inventory"; then printf "\t<li><a href='https://uva-bi-sdad.github.io/sdc.all_dev/${line}'> $line </a></li>\n" >> docs/index.html; fi; done
printf "</ol>\n" >> docs/index.html
rm docs/temp
- name: Commit the results
run: |
git config --local user.email "[email protected]"
git config --local user.name "GitHub Actions"
git add .
git commit -m "Automated manifest update"
- name: Push changes to current repository
uses: ad-m/github-push-action@master
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
branch: ${{ github.ref }}
- name: Remove github things # Required so that you can push to another directory
run: |
rm -rf .git
- name: Remove github submodules # Required so that you can push to another directory
run: |
rm .gitmodules
- name: Remove github workflow # Required so that you can push to another directory
run: |
rm -rf .github
- name: Push directory to another repository
uses: cpina/github-action-push-to-another-repository@main
env:
SSH_DEPLOY_KEY: ${{ secrets.SSH_DEPLOY_KEY }}
with:
source-directory: .
destination-github-username: 'uva-bi-sdad'
destination-repository-name: 'sdc.all'
target-branch: main
# Running the GitHub pages here so that you don't need to run runners twice
- name: Setup Pages
uses: actions/configure-pages@v3
- name: Upload artifact
uses: actions/upload-pages-artifact@v2
with:
path: '${{ github.workspace }}/docs'
- name: Deploy to GitHub Pages
id: deployment
uses: actions/deploy-pages@v2
- name: Generate a variables folder and export all the data there
run: mkdir -p variables
- name: Check out editor repository code
uses: actions/checkout@master
with:
repository: uva-bi-sdad/sdc.editor
path: ${{ github.workspace }}/sdc.editor
- name: List files in the repository (Useful for debugging)
run: |
ls -la ${{ github.workspace }}
- name: Install requirements for the editor
run: |
pip install -r ${{ github.repository }}/sdc.editor/code/requirements.txt
- name: Generate the variables data
run: |
python ${{ github.repository }}/sdc.editor/code/edit_99_combine_split_by_variable.py -i ${{ github.repository }} -o ${{ github.repository }}/variables
- name: Push directory to another repository
uses: cpina/github-action-push-to-another-repository@main
env:
SSH_DEPLOY_KEY: ${{ secrets.VARIABLES_DEPLOY_KEY }}
with:
source-directory: ${{ github.repository }}/variables
destination-github-username: 'uva-bi-sdad'
destination-repository-name: 'sdc.variables'
target-branch: main
on_fail:
if: ${{ failure() }} # still push the total public one to sdc.all on fail
needs: housekeeping
runs-on: ubuntu-latest
steps:
- name: Checkout repository and submodules
uses: actions/checkout@master
- name: Checkout latest submodules
run: git submodule update --init --recursive --remote
- name: Remove github things # Required so that you can push to another directory
run: |
rm -rf .git
- name: Remove github submodules # Required so that you can push to another directory
run: |
rm .gitmodules
- name: Remove github workflow # Required so that you can push to another directory
run: |
rm -rf .github
- name: Push directory to another repository
uses: cpina/github-action-push-to-another-repository@main
env:
SSH_DEPLOY_KEY: ${{ secrets.SSH_DEPLOY_KEY }}
with:
source-directory: .
destination-github-username: 'uva-bi-sdad'
destination-repository-name: 'sdc.all'
target-branch: main