forked from bionic-gpt/bionic-gpt
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathEarthfile
199 lines (170 loc) · 7.48 KB
/
Earthfile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
VERSION 0.7
FROM purtontech/rust-on-nails-devcontainer:1.1.18
ARG --global APP_EXE_NAME=web-ui
ARG --global OPERATOR_EXE_NAME=k8s-operator
ARG --global RABBITMQ_EXE_NAME=rabbit-mq
ARG --global PIPELINE_EXE_NAME=pipeline-job
ARG --global DBMATE_VERSION=2.2.0
# Folders
ARG --global DB_FOLDER=crates/db
ARG --global PIPELINE_FOLDER=crates/asset-pipeline
# Images with models
ARG --global EMBEDDINGS_IMAGE_NAME=ghcr.io/bionic-gpt/bionicgpt-embeddings-api:cpu-0.6
# This file builds the following containers
ARG --global APP_IMAGE_NAME=bionic-gpt/bionicgpt:latest
ARG --global MIGRATIONS_IMAGE_NAME=bionic-gpt/bionicgpt-db-migrations:latest
ARG --global PIPELINE_IMAGE_NAME=bionic-gpt/bionicgpt-pipeline-job:latest
ARG --global OPERATOR_IMAGE_NAME=bionic-gpt/bionicgpt-k8s-operator:latest
ARG --global RABBITMQ_IMAGE_NAME=bionic-gpt/bionicgpt-rabbitmq:latest
WORKDIR /build
USER vscode
dev:
BUILD +pull-request
# On github this check is performed directly by the action
#BUILD +integration-test
#BUILD +check-selenium-failure
pull-request:
BUILD +migration-container
BUILD +app-container
BUILD +operator-container
BUILD +pipeline-job-container
BUILD +rabbitmq-container
all:
BUILD +migration-container
BUILD +app-container
BUILD +operator-container
BUILD +pipeline-job-container
BUILD +rabbitmq-container
npm-deps:
COPY $PIPELINE_FOLDER/package.json $PIPELINE_FOLDER/package.json
COPY $PIPELINE_FOLDER/package-lock.json $PIPELINE_FOLDER/package-lock.json
COPY --dir $PIPELINE_FOLDER/patches $PIPELINE_FOLDER/patches
RUN cd $PIPELINE_FOLDER && npm install
SAVE ARTIFACT $PIPELINE_FOLDER/node_modules
npm-build:
FROM +npm-deps
COPY $PIPELINE_FOLDER $PIPELINE_FOLDER
COPY +npm-deps/node_modules $PIPELINE_FOLDER/node_modules
COPY --dir crates/ui-pages crates/ui-pages
COPY --dir crates/daisy-rsx crates/daisy-rsx
RUN cd $PIPELINE_FOLDER && npm run release
SAVE ARTIFACT $PIPELINE_FOLDER/dist
pipeline-job-container:
FROM scratch
COPY +build/$PIPELINE_EXE_NAME pipeline-job
ENTRYPOINT ["./pipeline-job"]
SAVE IMAGE --push $PIPELINE_IMAGE_NAME
rabbitmq-container:
FROM scratch
COPY +build/$RABBITMQ_EXE_NAME rabbit-mq
ENTRYPOINT ["./rabbit-mq"]
SAVE IMAGE --push $RABBITMQ_IMAGE_NAME
build-web-ui:
# Copy in all our crates
COPY --dir crates crates
RUN rm -rf crates/rabbit-mq crates/k8s-operator crates/pipeline-job
COPY --dir Cargo.lock Cargo.toml .
COPY --dir +npm-build/dist $PIPELINE_FOLDER/
# Install cargo-binstall, which makes it easier to install other
# cargo extensions like cargo-leptos
RUN wget https://github.com/cargo-bins/cargo-binstall/releases/latest/download/cargo-binstall-x86_64-unknown-linux-musl.tgz
RUN tar -xvf cargo-binstall-x86_64-unknown-linux-musl.tgz
RUN cp cargo-binstall /usr/local/cargo/bin
RUN rustup target add wasm32-unknown-unknown
# Install cargo-leptos
RUN cargo binstall cargo-leptos -y
# We need to run inside docker as we need postgres running for cornucopia
ARG DATABASE_URL=postgresql://postgres:testpassword@localhost:5432/postgres?sslmode=disable
USER root
WITH DOCKER \
--pull ankane/pgvector
RUN docker run -d --rm --network=host -e POSTGRES_PASSWORD=testpassword ankane/pgvector \
&& dbmate --wait --migrations-dir $DB_FOLDER/migrations up \
&& cargo leptos build --release -vv
END
SAVE ARTIFACT target/release/$APP_EXE_NAME
SAVE ARTIFACT target/site
build:
# Copy in all our crates
COPY --dir crates crates
RUN rm -rf crates/axum-server crates/web-ui crates/asset-pipeline crates/daisy-rsx crates/ui-pages
COPY --dir Cargo.lock Cargo.toml .
# We need to run inside docker as we need postgres running for cornucopia
ARG DATABASE_URL=postgresql://postgres:testpassword@localhost:5432/postgres?sslmode=disable
USER root
WITH DOCKER \
--pull ankane/pgvector
RUN docker run -d --rm --network=host -e POSTGRES_PASSWORD=testpassword ankane/pgvector \
&& dbmate --wait --migrations-dir $DB_FOLDER/migrations up \
&& cargo build --release --target x86_64-unknown-linux-musl
END
SAVE ARTIFACT target/x86_64-unknown-linux-musl/release/$PIPELINE_EXE_NAME
SAVE ARTIFACT target/x86_64-unknown-linux-musl/release/$RABBITMQ_EXE_NAME
SAVE ARTIFACT target/x86_64-unknown-linux-musl/release/$OPERATOR_EXE_NAME
migration-container:
FROM alpine
RUN apk add --no-cache \
curl \
postgresql-client \
tzdata
RUN curl -OL https://github.com/amacneil/dbmate/releases/download/v$DBMATE_VERSION/dbmate-linux-amd64 \
&& mv ./dbmate-linux-amd64 /usr/bin/dbmate \
&& chmod +x /usr/bin/dbmate
COPY --dir $DB_FOLDER .
CMD dbmate up
SAVE IMAGE --push $MIGRATIONS_IMAGE_NAME
# To test this locally run
# docker run -it --rm -e APP_DATABASE_URL=$APP_DATABASE_URL -p 7703:7703 bionic-gpt/bionicgpt:latest
app-container:
FROM debian:bookworm-slim
COPY +build-web-ui/$APP_EXE_NAME web-ui
COPY +build-web-ui/site site
# Place assets in a build folder as that's where statics is expecting them.
COPY --dir +npm-build/dist /build/$PIPELINE_FOLDER/
COPY --dir $PIPELINE_FOLDER/images /build/$PIPELINE_FOLDER/images
ENV LEPTOS_SITE_ADDR="0.0.0.0:7703"
ENV LEPTOS_SITE_ROOT="site"
ENTRYPOINT ["./web-ui"]
SAVE IMAGE --push $APP_IMAGE_NAME
# We've got a Kubernetes operator
operator-container:
FROM scratch
COPY +build/$OPERATOR_EXE_NAME k8s-operator
ENTRYPOINT ["./k8s-operator", "operator"]
SAVE IMAGE --push $OPERATOR_IMAGE_NAME
# Embeddings container - download models from huggungface
embeddings-container-base:
FROM purtontech/rust-on-nails-devcontainer:1.1.17
RUN sudo apt install -y python3-venv python3-pip
RUN sudo pip install -U "huggingface_hub[cli]" --break-system-packages
RUN sudo huggingface-cli download --cache-dir ./data BAAI/bge-small-en-v1.5 1_Pooling/config.json
RUN sudo huggingface-cli download --cache-dir ./data BAAI/bge-small-en-v1.5 model.safetensors
RUN sudo huggingface-cli download --cache-dir ./data BAAI/bge-small-en-v1.5 config.json
RUN sudo huggingface-cli download --cache-dir ./data BAAI/bge-small-en-v1.5 tokenizer.json
SAVE ARTIFACT ./data
embeddings-container:
FROM ghcr.io/huggingface/text-embeddings-inference:cpu-0.6
COPY +embeddings-container-base/data /data
CMD ["--json-output", "--model-id", "BAAI/bge-small-en-v1.5"]
SAVE IMAGE --push $EMBEDDINGS_IMAGE_NAME
build-cli-linux:
COPY --dir crates/k8s-operator .
RUN cd k8s-operator && cargo build --release
SAVE ARTIFACT k8s-operator/target/release/k8s-operator AS LOCAL ./bionic-cli-linux
build-cli-osx:
FROM joseluisq/rust-linux-darwin-builder:1.76.0
COPY --dir crates/k8s-operator .
RUN cd k8s-operator \
&& CC=o64-clang \
CXX=o64-clang++ \
cargo build --release --target x86_64-apple-darwin
SAVE ARTIFACT k8s-operator/target/x86_64-apple-darwin/release/k8s-operator AS LOCAL ./bionic-cli-darwin
build-cli-windows:
RUN sudo apt update && sudo apt upgrade -y
RUN sudo apt install -y g++-mingw-w64-x86-64
RUN rustup target add x86_64-pc-windows-gnu
RUN rustup toolchain install stable-x86_64-pc-windows-gnu
COPY --dir crates/k8s-operator .
RUN cd k8s-operator \
&& cargo build --release --target x86_64-pc-windows-gnu
SAVE ARTIFACT k8s-operator/target/x86_64-pc-windows-gnu/release/k8s-operator.exe AS LOCAL ./bionic-cli-windows.exe