-
Notifications
You must be signed in to change notification settings - Fork 3
/
Copy pathdodo.py
145 lines (125 loc) · 6.14 KB
/
dodo.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
from core.config import Config
def task_create_k8s_namespaces():
return {
"actions": [
"kubectl create namespace model-factory-pipelines --dry-run=client -o yaml | kubectl apply -f -",
"kubectl create namespace model-factory-services --dry-run=client -o yaml | kubectl apply -f -",
]
}
def task_model_factory_frontend():
return {
"actions": [
"docker pull {}/model_factory_base_image".format(Config.DOCKER_REGISTRY),
"docker build --build-arg DOCKER_REGISTRY={} . -f services/model_factory_frontend/Dockerfile -t {}/model-factory-frontend".format(
Config.DOCKER_REGISTRY, Config.DOCKER_REGISTRY
),
"docker push {}/model-factory-frontend".format(Config.DOCKER_REGISTRY),
"DOCKER_REGISTRY={} envsubst < services/model_factory_frontend/deployment/deployment.yaml | kubectl apply -f -".format(Config.DOCKER_REGISTRY),
],
}
def task_model_factory_dashboard():
return {
"actions": [
"docker pull {}/model_factory_base_image".format(Config.DOCKER_REGISTRY),
"docker build --build-arg DOCKER_REGISTRY={} . -f services/model_factory_dashboard/Dockerfile -t {}/model-factory-dashboard".format(
Config.DOCKER_REGISTRY, Config.DOCKER_REGISTRY
),
"docker push {}/model-factory-dashboard".format(Config.DOCKER_REGISTRY),
"DOCKER_REGISTRY={} envsubst < services/model_factory_dashboard/deployment/deployment.yaml | kubectl apply -f -".format(Config.DOCKER_REGISTRY),
],
}
def task_model_factory_execution_syncer():
return {
"actions": [
"docker pull {}/model_factory_base_image".format(Config.DOCKER_REGISTRY),
"docker build --build-arg DOCKER_REGISTRY={} . -f services/model_factory_execution_syncer/Dockerfile -t {}/model-factory-execution-syncer".format(
Config.DOCKER_REGISTRY, Config.DOCKER_REGISTRY
),
"docker push {}/model-factory-execution-syncer".format(Config.DOCKER_REGISTRY),
"DOCKER_REGISTRY={} envsubst < services/model_factory_execution_syncer/deployment/deployment.yaml | kubectl apply -f -".format(Config.DOCKER_REGISTRY),
],
}
def task_model_factory_autohide_service():
return {
"actions": [
"docker pull {}/model_factory_base_image".format(Config.DOCKER_REGISTRY),
"docker build --build-arg DOCKER_REGISTRY={} . -f services/model_factory_autohide_service/Dockerfile -t {}/model-factory-autohide-service".format(
Config.DOCKER_REGISTRY, Config.DOCKER_REGISTRY
),
"docker push {}/model-factory-autohide-service".format(Config.DOCKER_REGISTRY),
"DOCKER_REGISTRY={} envsubst < services/model_factory_autohide_service/deployment/deployment.yaml | kubectl apply -f -".format(Config.DOCKER_REGISTRY),
],
}
def task_model_factory_trigger_service():
return {
"actions": [
"docker pull {}/model_factory_base_image".format(Config.DOCKER_REGISTRY),
"docker build --build-arg DOCKER_REGISTRY={} . -f services/model_factory_trigger_service/Dockerfile -t {}/model-factory-trigger-service".format(
Config.DOCKER_REGISTRY, Config.DOCKER_REGISTRY
),
"docker push {}/model-factory-trigger-service".format(Config.DOCKER_REGISTRY),
"DOCKER_REGISTRY={} envsubst < services/model_factory_trigger_service/deployment/deployment.yaml | kubectl apply -f -".format(Config.DOCKER_REGISTRY),
],
}
def task_model_factory_model_downloader():
return {
"actions": [
"docker pull {}/model_factory_base_image".format(Config.DOCKER_REGISTRY),
"docker build --build-arg DOCKER_REGISTRY={} . -f services/model_factory_model_downloader/Dockerfile -t {}/model-factory-model-downloader".format(
Config.DOCKER_REGISTRY, Config.DOCKER_REGISTRY
),
"docker push {}/model-factory-model-downloader".format(Config.DOCKER_REGISTRY),
],
}
def task_server():
return {
"actions": None,
"task_dep": [
"create_k8s_namespaces",
"model_factory_frontend",
"model_factory_frontend",
"model_factory_execution_syncer",
"model_factory_autohide_service",
"model_factory_trigger_service",
"model_factory_model_downloader",
],
}
def task_client():
return {
"actions": [
"echo \033[93mSetting up model factory client on your devbox...\033[0m",
"pip3 install awscli boto3 click dataclasses docker git+https://github.com/kubernetes-client/[email protected] gitpython jsonpickle pudb pymongo python-dateutil pytz pyyaml tabulate thrift treelib ansible_runner croniter slackclient",
"PYTHONPATH=`pwd` python3 cli/mf.py dev install-alias",
"echo \033[92mModel factory has been successully installed. Please restart your shell before using model factory!\033[0m",
]
}
def task_build_base_image():
return {
"actions": [
(
"docker build . -f docker/model_factory_base_image.dockerfile "
"--build-arg MONGO_DB_ENDPOINT={} "
"--build-arg MF_FRONTEND_ENDPOINT={} "
"--build-arg S3_ENDPOINT={} "
"--build-arg S3_BUCKET={} "
"--build-arg DOCKER_REGISTRY={} "
"--build-arg AWS_ACCESS_KEY_ID={} "
"--build-arg AWS_SECRET_ACCESS_KEY={} "
"--build-arg STORAGE_CLASS={} "
"-t {}/model_factory_base_image"
).format(
Config.MONGO_DB_ENDPOINT,
Config.MF_FRONTEND_ENDPOINT,
Config.S3_ENDPOINT,
Config.S3_BUCKET,
Config.DOCKER_REGISTRY,
Config.AWS_ACCESS_KEY_ID,
Config.AWS_SECRET_ACCESS_KEY,
Config.STORAGE_CLASS,
Config.DOCKER_REGISTRY,
),
"docker push {}/model_factory_base_image".format(
Config.DOCKER_REGISTRY,
),
]
}