-
Notifications
You must be signed in to change notification settings - Fork 2
136 lines (109 loc) · 3.9 KB
/
train_autoencoder.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
# This is a basic workflow to help you get started with Actions
name: Autoencoder Training
# Controls when the action will run.
on:
pull_request:
branches: [ main ]
paths:
- '.github/workflows/train_autoencoder.yml'
- 'src/**'
- 'dvc.yaml'
- 'params.yaml'
- 'scripts/**'
- 'requirements.txt'
- 'scripts/*reports.sh'
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
jobs:
build:
runs-on: [self-hosted,gpu,training]
container:
image: etheredgeb/pokegen:cicd
env:
AWS_ACCESS_KEY_ID: ${{ secrets.MINIO_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.MINIO_AWS_SECRET_ACCESS_KEY }}
repo_token: ${{ secrets.GITHUB_TOKEN }}
SHOULD_TQDM: 0 # disables tqdm in 'src/train_ae.py'
options: --gpus all --shm-size=1G
timeout-minutes: 1440 # Increased job runtime since ML is slow
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v2
with:
ref: ${{ github.head_ref }} # https://github.com/stefanzweifel/git-auto-commit-action
- name: Install as local package
run: pip install -e .
- uses: actions/setup-node@v2
with:
node-version: '16'
# - uses: actions/setup-python@v2
# with:
# python-version: '3.x'
- uses: iterative/setup-cml@v1
- name: pull_data
run: dvc pull data/external/sprites.dvc
- name: pull_repros
continue-on-error: true
run: dvc pull --run-cache train@ae_dense train@ae_conv train@vae_dense train@vae_conv train@vae
- name: train_ae_dense
run: |
dvc repro train@ae_dense
dvc push train@ae_dense
- name: report_ae_dense
run: |
scripts/autoencoder_report.sh ae dense >> report.md
cml-send-github-check report.md
- name: train_ae_conv
run: |
dvc repro train@ae_conv
dvc push train@ae_conv
- name: report_ae_conv
run: |
scripts/autoencoder_report.sh ae conv >> report.md
cml-send-github-check report.md
- name: train_vae_dense
run: |
dvc repro train@vae_dense
dvc push train@vae_dense
- name: report_vae_dense
run: |
scripts/autoencoder_report.sh vae dense >> report.md
cml-send-github-check report.md
- name: train_vae_conv
run: |
dvc repro train@vae_conv
dvc push train@vae_conv
- name: report_vae_conv
run: |
scripts/autoencoder_report.sh vae conv >> report.md
cml-send-github-check report.md
- name: train_vae
run: |
dvc repro train@vae
dvc push train@vae
- name: report_vae
run: |
scripts/autoencoder_report.sh vae vae >> report.md
cml-send-github-check report.md
- run: |
git config user.name github-actions
git config user.email [email protected]
git pull
- uses: stefanzweifel/git-auto-commit-action@v4
with:
commit_message: "Latest Github Action DVC Run"
file_pattern: dvc.lock reports/*/*/summary.txt reports/*/*/logs.json
- name: main_push
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: dvc push -r s3
- name: report_all
if: steps.auto-commit-action.outputs.changes_detected == 'true'
run: scripts/all_autoencoder_reports.sh > report.md
- name: no_changes
if: steps.auto-commit-action.outputs.changes_detected == 'false'
run: echo "No changes" > report.md
- name: final_report
run: cml-send-comment report.md