-
Notifications
You must be signed in to change notification settings - Fork 18
/
Makefile
464 lines (391 loc) · 17 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
COMPONENTLIST := gateway-mt authservice linksharing
#
# Common
#
.PHONY: help
help:
@awk 'BEGIN { \
FS = ":.*##"; \
printf "\nUsage:\n make \033[36m<target>\033[0m\n" \
} \
/^[a-zA-Z_-]+:.*?##/ { \
printf " \033[36m%-28s\033[0m %s\n", $$1, $$2 \
} \
/^##@/ { \
printf "\n\033[1m%s\033[0m\n", substr($$0, 5) \
}' $(MAKEFILE_LIST)
.DEFAULT_GOAL := help
#
# Public Jenkins (commands below are used for local development and/or public Jenkins)
#
#@ Local development/Public Jenkins/Helpers
.PHONY: install-dev-dependencies
install-dev-dependencies: badgerauth-install-dependencies ## install-dev-dependencies assumes Go and cURL are installed
# Storj-specific:
go install github.com/storj/ci/check-mod-tidy@latest
go install github.com/storj/ci/check-copyright@latest
go install github.com/storj/ci/check-large-files@latest
go install github.com/storj/ci/check-imports@latest
go install github.com/storj/ci/check-peer-constraints@latest
go install github.com/storj/ci/check-atomic-align@latest
go install github.com/storj/ci/check-monkit@latest
go install github.com/storj/ci/check-errs@latest
go install github.com/storj/ci/check-deferloop@latest
go install github.com/storj/ci/check-downgrades@latest
go install github.com/storj/ci/storj-release@latest
# staticcheck:
go install honnef.co/go/tools/cmd/staticcheck@latest
# golangci-lint:
go install github.com/golangci/golangci-lint/cmd/[email protected]
# shellcheck (TODO(artur,sean): Windows)
ifneq ($(shell which apt-get),)
sudo apt-get install -y shellcheck
else ifneq ($(shell which brew),)
brew install shellcheck
else
$(error Can't install shellcheck without a supported package manager)
endif
go install github.com/google/[email protected]
.PHONY: badgerauth-install-dependencies
badgerauth-install-dependencies:
go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
go install storj.io/drpc/cmd/protoc-gen-go-drpc@latest
ifneq ($(shell which apt-get),)
sudo apt-get install -y protobuf-compiler
else ifneq ($(shell which brew),)
brew install protobuf
else
$(error Can't install protobuf without a supported package manager)
endif
.PHONY: badgerauth-format-protobufs
badgerauth-format-protobufs:
ifeq ($(shell which clang-format),)
# If clang-format isn't found, we want to install it first:
ifneq ($(shell which apt-get),)
sudo apt-get install -y clang-format
else ifneq ($(shell which brew),)
brew install clang-format
else
$(error Can't install clang-format without a supported package manager)
endif
endif
clang-format -i pkg/auth/badgerauth/pb/badgerauth.proto
clang-format -i pkg/auth/badgerauth/pb/badgerauth_admin.proto
.PHONY: bump-code-dependencies
bump-code-dependencies:
go get storj.io/gateway@main && go mod tidy && cd testsuite && go mod tidy && \
go get storj.io/storj@latest && go mod tidy
.PHONY: install-hooks
install-hooks: ## Install helpful Git hooks
ln -s ../../githooks/pre-commit .git/hooks/pre-commit
##@ Local development/Public Jenkins/Lint
GOLANGCI_LINT_CONFIG ?= ../ci/.golangci.yml
GOLANGCI_LINT_CONFIG_TESTSUITE ?= ../../ci/.golangci.yml
.PHONY: lint
lint: ## Lint
check-mod-tidy
check-copyright
check-large-files
check-imports -race ./...
check-peer-constraints -race
check-atomic-align ./...
check-monkit ./...
check-errs ./...
check-deferloop ./...
staticcheck ./...
golangci-lint run --print-resources-usage --config ${GOLANGCI_LINT_CONFIG}
check-downgrades
go-licenses check --ignore "storj.io/dotworld,storj.io/edge" ./...
# A bit of an explanation around this shellcheck command:
# * Find all scripts recursively that have the .sh extension, except for "testsuite@tmp" which Jenkins creates temporarily
# * Use + instead of \ so find returns a non-zero exit if any invocation of shellcheck returns a non-zero exit
find . -path ./testsuite@tmp -prune -o -name "*.sh" -type f -exec "shellcheck" "-x" "--format=gcc" {} +;
# Execute lint-testsuite in testsuite directory:
$(MAKE) -C testsuite -f ../Makefile lint-testsuite
.PHONY: lint-testsuite
lint-testsuite: ## Lint testsuite
check-imports -race ./...
check-atomic-align ./...
check-monkit ./...
check-errs ./...
check-deferloop ./...
staticcheck ./...
golangci-lint run --print-resources-usage --config ${GOLANGCI_LINT_CONFIG_TESTSUITE}
go-licenses check --ignore "storj.io/dotworld,storj.io/edge" ./...
##@ Local development/Public Jenkins/Cross-Vet
.PHONY: cross-vet
cross-vet: ## Cross-Vet
GOOS=linux GOARCH=386 go vet ./...
GOOS=linux GOARCH=amd64 go vet ./...
GOOS=linux GOARCH=arm go vet ./...
GOOS=linux GOARCH=arm64 go vet ./...
GOOS=freebsd GOARCH=386 go vet ./...
GOOS=freebsd GOARCH=amd64 go vet ./...
GOOS=freebsd GOARCH=arm go vet ./...
GOOS=freebsd GOARCH=arm64 go vet ./...
GOOS=windows GOARCH=386 go vet ./...
GOOS=windows GOARCH=amd64 go vet ./...
# TODO(artur,sean): find out if we will be able to enable these:
# GOOS=windows GOARCH=arm go vet ./...
# GOOS=windows GOARCH=arm64 go vet ./...
# Use kqueue to avoid using Cgo for verification.
GOOS=darwin GOARCH=amd64 go vet -tags kqueue ./...
GOOS=darwin GOARCH=arm64 go vet -tags kqueue ./...
##@ Local development/Public Jenkins/Test
JSON ?= false
SHORT ?= true
SKIP_TESTSUITE ?= false
.PHONY: test
test: test-testsuite ## Test
go test -json=${JSON} -p 16 -parallel 4 -race -short=${SHORT} -timeout 10m -vet=off ./...
.PHONY: test-testsuite
test-testsuite: ## Test testsuite
ifeq (${SKIP_TESTSUITE},false)
# Execute test-testsuite-do in testsuite directory:
$(MAKE) -C testsuite -f ../Makefile test-testsuite-do
endif
.PHONY: test-testsuite-do
test-testsuite-do:
go vet ./...
go test -json=${JSON} -p 16 -parallel 4 -race -short=${SHORT} -timeout 10m -vet=off ./...
##@ Local development/Public Jenkins/Verification
.PHONY: verify
verify: lint cross-vet test ## Execute pre-commit verification
#
# Private Jenkins (commands below are used for releases/private Jenkins)
#
##@ Release/Private Jenkins/Build
GO_VERSION ?= 1.23.4
BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD | sed "s!/!-!g")
ifeq (${BRANCH_NAME},main)
TAG := $(shell git rev-parse --short HEAD)-go${GO_VERSION}
BRANCH_NAME :=
else
TAG := $(shell git rev-parse --short HEAD)-${BRANCH_NAME}-go${GO_VERSION}
ifneq ($(shell git describe --tags --exact-match --match "v[0-9]*\.[0-9]*\.[0-9]*"),)
LATEST_STABLE_TAG := latest
endif
endif
DOCKER_BUILD := docker build --build-arg TAG=${TAG}
LATEST_DEV_TAG := dev
.PHONY: images
images: gateway-mt-image authservice-image linksharing-image ## Build Docker images
@echo Built version: ${TAG}
.PHONY: gateway-mt-image
gateway-mt-image: ## Build gateway-mt Docker image
${DOCKER_BUILD} --platform linux/amd64 --pull=true -t storjlabs/gateway-mt:${TAG}-amd64 \
-f cmd/gateway-mt/Dockerfile .
${DOCKER_BUILD} --platform linux/arm/v6 --pull=true -t storjlabs/gateway-mt:${TAG}-arm32v6 \
--build-arg=GOARCH=arm \
--build-arg=DOCKER_ARCH=arm32v6 \
-f cmd/gateway-mt/Dockerfile .
${DOCKER_BUILD} --platform linux/arm64 --pull=true -t storjlabs/gateway-mt:${TAG}-arm64v8 \
--build-arg=GOARCH=arm64 \
--build-arg=DOCKER_ARCH=arm64v8 \
-f cmd/gateway-mt/Dockerfile .
docker tag storjlabs/gateway-mt:${TAG}-amd64 storjlabs/gateway-mt:${LATEST_DEV_TAG}
.PHONY: authservice-image
authservice-image: ## Build authservice Docker image
${DOCKER_BUILD} --platform linux/amd64 --pull=true -t storjlabs/authservice:${TAG}-amd64 \
-f cmd/authservice/Dockerfile .
${DOCKER_BUILD} --platform linux/arm/v6 --pull=true -t storjlabs/authservice:${TAG}-arm32v6 \
--build-arg=GOARCH=arm \
--build-arg=DOCKER_ARCH=arm32v6 \
-f cmd/authservice/Dockerfile .
${DOCKER_BUILD} --platform linux/arm64 --pull=true -t storjlabs/authservice:${TAG}-arm64v8 \
--build-arg=GOARCH=arm64 \
--build-arg=DOCKER_ARCH=arm64v8 \
-f cmd/authservice/Dockerfile .
docker tag storjlabs/authservice:${TAG}-amd64 storjlabs/authservice:${LATEST_DEV_TAG}
.PHONY: linksharing-image
linksharing-image: ## Build linksharing Docker image
${DOCKER_BUILD} --platform linux/amd64 --pull=true -t storjlabs/linksharing:${TAG}-amd64 \
-f cmd/linksharing/Dockerfile .
${DOCKER_BUILD} --platform linux/arm/v6 --pull=true -t storjlabs/linksharing:${TAG}-arm32v6 \
--build-arg=GOARCH=arm --build-arg=DOCKER_ARCH=arm32v6 \
-f cmd/linksharing/Dockerfile .
${DOCKER_BUILD} --platform linux/arm64 --pull=true -t storjlabs/linksharing:${TAG}-arm64v8 \
--build-arg=GOARCH=arm64 --build-arg=DOCKER_ARCH=arm64v8 \
-f cmd/linksharing/Dockerfile .
docker tag storjlabs/linksharing:${TAG}-amd64 storjlabs/linksharing:${LATEST_DEV_TAG}
.PHONY: binaries
binaries: ${BINARIES} ## Build gateway-mt, authservice, and linksharing binaries
for C in ${COMPONENTLIST}; do \
# freebsd/amd64 target is currently skipped: https://github.com/storj/gateway-st/issues/62 \
CGO_ENABLED=0 storj-release \
--components "cmd/$$C" \
--build-tags kqueue \
--go-version "${GO_VERSION}" \
--branch "${BRANCH_NAME}" \
--skip-osarches "freebsd/amd64" || exit $$? \
; done
.PHONY: push-images
push-images: ## Push Docker images to Docker Hub
# images have to be pushed before a manifest can be created
for c in ${COMPONENTLIST}; do \
docker push storjlabs/$$c:${TAG}-amd64 \
&& docker push storjlabs/$$c:${TAG}-arm32v6 \
&& docker push storjlabs/$$c:${TAG}-arm64v8 \
&& for t in ${TAG} ${LATEST_DEV_TAG} ${LATEST_STABLE_TAG}; do \
docker manifest create storjlabs/$$c:$$t \
storjlabs/$$c:${TAG}-amd64 \
storjlabs/$$c:${TAG}-arm32v6 \
storjlabs/$$c:${TAG}-arm64v8 \
&& docker manifest annotate storjlabs/$$c:$$t storjlabs/$$c:${TAG}-amd64 --os linux --arch amd64 \
&& docker manifest annotate storjlabs/$$c:$$t storjlabs/$$c:${TAG}-arm32v6 --os linux --arch arm --variant v6 \
&& docker manifest annotate storjlabs/$$c:$$t storjlabs/$$c:${TAG}-arm64v8 --os linux --arch arm64 --variant v8 \
&& docker manifest push --purge storjlabs/$$c:$$t \
; done \
; done
.PHONY: binaries-upload
binaries-upload: ## Upload release binaries to GCS
cd "release/${TAG}"; for f in *; do \
c="$${f%%_*}" \
&& if [ "$${f##*.}" != "$${f}" ]; then \
ln -s "$${f}" "$${f%%_*}.$${f##*.}" \
&& zip "$${f}.zip" "$${f%%_*}.$${f##*.}" \
&& rm "$${f%%_*}.$${f##*.}" \
; else \
ln -sf "$${f}" "$${f%%_*}" \
&& zip "$${f}.zip" "$${f%%_*}" \
&& rm "$${f%%_*}" \
; fi \
; done
cd "release/${TAG}" \
&& sha256sum *.zip > sha256sums \
&& gsutil -m cp -r *.zip sha256sums "gs://storj-v3-alpha-builds/${TAG}/"
##@ Release/Private Jenkins/Clean
.PHONY: clean
clean: clean-binaries clean-images ## Remove local release binaries and local Docker images
.PHONY: clean-binaries
clean-binaries: ## Remove local release binaries
rm -rf release
.PHONY: clean-images
clean-images:
-docker rmi -f $(shell docker images -q "storjlabs/gateway-mt:${TAG}-*")
-docker rmi -f $(shell docker images -q "storjlabs/authservice:${TAG}-*")
-docker rmi -f $(shell docker images -q "storjlabs/linksharing:${TAG}-*")
##@ Local development/Public Jenkins/Integration Test
BUILD_NUMBER ?= ${TAG}
.PHONY: integration-run
integration-run: integration-env-start integration-all-tests ## Start the integration environment and run all tests
.PHONY: integration-env-start
integration-env-start: integration-checkout integration-image-build integration-network-create integration-services-start ## Start the integration environment
.PHONY: integration-env-stop
integration-env-stop: ## Stop all running services in the integration environment
-docker stop --time=1 $$(docker ps -qf network=integration-network-${BUILD_NUMBER})
.PHONY: integration-env-clean
integration-env-clean:
-docker rm $$(docker ps -aqf network=integration-network-${BUILD_NUMBER})
-docker rmi $$(docker image ls -qf label=build=${BUILD_NUMBER})
-docker rmi redis:latest
-docker rmi postgres:latest
-docker rmi storjlabs/gateway-mint:latest
-docker rmi storjlabs/splunk-s3-tests:latest
-docker compose down
-rm -r volumes
-rm -rf gateway-st storj
-rm -rf edge.Dockerfile storj.Dockerfile docker-compose.yaml
.PHONY: integration-env-purge
integration-env-purge: integration-env-stop integration-env-clean integration-network-remove ## Purge the integration environment
.PHONY: integration-env-logs
integration-env-logs: ## Retrieve logs from integration services
-docker logs integration-authservice-${BUILD_NUMBER}
-docker logs integration-gateway-${BUILD_NUMBER}
.PHONY: integration-all-tests
integration-all-tests: integration-gateway-st-tests integration-mint-tests integration-splunk-tests ## Run all integration tests (environment needs to be started first)
# note: umask 0000 is needed for rclone tests so files can be cleaned up.
.PHONY: integration-gateway-st-tests
integration-gateway-st-tests: ## Run gateway-st test suite (environment needs to be started first)
$$(docker compose exec -T satellite-api storj-up credentials --s3 -e -a http://authservice:20000 -s satellite-api:7777) && \
docker run \
--cap-add SYS_ADMIN --device /dev/fuse --security-opt apparmor:unconfined \
--network integration-network-${BUILD_NUMBER} \
-e AWS_ENDPOINT=https://gateway:20011 -e "AWS_ACCESS_KEY_ID=$$AWS_ACCESS_KEY_ID" -e "AWS_SECRET_ACCESS_KEY=$$AWS_SECRET_ACCESS_KEY" \
-v $$PWD:/build \
-w /build \
--name integration-gateway-st-tests-${BUILD_NUMBER}-$$TEST \
--entrypoint /bin/bash \
--rm storjlabs/ci:latest \
-c "umask 0000; scripts/run-integration-tests.sh $$TEST" \
.PHONY: integration-mint-tests
integration-mint-tests: ## Run mint test suite (environment needs to be started first)
$$(docker compose exec -T satellite-api storj-up credentials --s3 -e -a http://authservice:20000 -s satellite-api:7777) && \
docker run \
--network integration-network-${BUILD_NUMBER} \
-e SERVER_ENDPOINT=gateway:20010 -e "ACCESS_KEY=$$AWS_ACCESS_KEY_ID" -e "SECRET_KEY=$$AWS_SECRET_ACCESS_KEY" -e ENABLE_HTTPS=0 \
--name integration-mint-tests-${BUILD_NUMBER}-$$TEST \
--rm storjlabs/gateway-mint:latest $$TEST
.PHONY: integration-splunk-tests
integration-splunk-tests: ## Run splunk test suite (environment needs to be started first)
$$(docker compose exec -T satellite-api storj-up credentials --s3 -e -a http://authservice:20000 -s satellite-api:7777) && \
docker run \
--network integration-network-${BUILD_NUMBER} \
-e ENDPOINT=gateway:20010 -e "AWS_ACCESS_KEY_ID=$$AWS_ACCESS_KEY_ID" -e "AWS_SECRET_ACCESS_KEY=$$AWS_SECRET_ACCESS_KEY" -e SECURE=0 \
--name integration-splunk-tests-${BUILD_NUMBER} \
--rm storjlabs/splunk-s3-tests:latest
.PHONY: integration-checkout
integration-checkout:
git clone --filter blob:none --depth 1 --no-tags --no-checkout https://github.com/storj/gateway-st gateway-st
cd gateway-st && \
git config core.sparsecheckout true && \
echo "testsuite/integration" >> .git/info/sparse-checkout && \
git checkout
.PHONY: integration-image-build
integration-image-build:
for C in gateway-mt authservice; do \
CGO_ENABLED=0 ./scripts/build-image.sh $$C ${BUILD_NUMBER} ${GO_VERSION} \
; done
storj-up init minimal,db && \
storj-up build remote github minimal -s && \
docker compose -p storj-up-integration build
.PHONY: integration-network-create
integration-network-create:
docker network create integration-network-${BUILD_NUMBER}
.PHONY: integration-network-remove
integration-network-remove:
-docker network remove integration-network-${BUILD_NUMBER}
.PHONY: integration-services-start
integration-services-start:
storj-up network set minimal,db integration-network-${BUILD_NUMBER} && \
storj-up network unset minimal,db default && \
storj-up env setenv satellite-api STORJ_CONSOLE_SIGNUP_ACTIVATION_CODE_ENABLED=false && \
storj-up env setenv satellite-api STORJ_METAINFO_USE_BUCKET_LEVEL_OBJECT_VERSIONING=true && \
storj-up env setenv satellite-api STORJ_METAINFO_OBJECT_LOCK_ENABLED=true && \
storj-up env set storagenode STORJUP_AUTHSERVICE=http://authservice:20000 && \
docker compose up -d && \
storj-up health
docker run \
--network integration-network-${BUILD_NUMBER} --network-alias authservice \
--name integration-authservice-${BUILD_NUMBER} \
--rm -d storjlabs/authservice:${BUILD_NUMBER} run \
--listen-addr 0.0.0.0:20000 \
--drpc-listen-addr 0.0.0.0:20002 \
--allowed-satellites $$(docker compose exec -T satellite-api storj-up util node-id /var/lib/storj/.local/share/storj/identity/satellite-api/identity.cert)@satellite-api:7777 \
--auth-token super-secret \
--endpoint http://gateway:20010 \
--kv-backend badger://
mkdir -p volumes/gateway
openssl req \
-x509 \
-newkey rsa:4096 \
-keyout volumes/gateway/cert.key \
-out volumes/gateway/cert.crt \
-nodes \
-subj '/CN=gateway' \
-addext "subjectAltName = DNS:gateway"
docker run \
--network integration-network-${BUILD_NUMBER} --network-alias gateway \
--name integration-gateway-${BUILD_NUMBER} \
--volume $$PWD/volumes/gateway:/cert:ro \
--rm -d storjlabs/gateway-mt:${BUILD_NUMBER} run \
--server.address 0.0.0.0:20010 \
--server.address-tls 0.0.0.0:20011 \
--auth.base-url http://authservice:20000 \
--auth.token super-secret \
--domain-name gateway \
--insecure-log-all \
--cert-dir /cert \
--insecure-disable-tls=false \
--s3compatibility.fully-compatible-listing \
--s3compatibility.disable-copy-object=false