From ca2c4e89d8d4995e71c6261c7ad5ae4f3d7dd64c Mon Sep 17 00:00:00 2001 From: Alec Thomas Date: Wed, 18 Dec 2024 19:50:45 +1100 Subject: [PATCH] chore: speed up parts of the build This should have a fairly decent cumulative effect. Before: ``` $ time just build-zips just build-zips 0.20s user 0.09s system 22% cpu 1.287 total $ time just go2proto just go2proto 0.83s user 0.09s system 84% cpu 1.090 total $ time just build ftl 2>&1 | grep Running ===> Running recipe `go2proto`... ===> Running recipe `build-protos`... ===> Running recipe `build-zips`... ===> Running recipe `pnpm-install`... ===> Running recipe `build-frontend`... ===> Running recipe `build`... ===> Running recipe `go2proto`... ===> Running recipe `build-protos`... ===> Running recipe `build-zips`... ===> Running recipe `build-without-frontend`... ===> Running recipe `build-zips`... ===> Running recipe `go2proto`... ===> Running recipe `build-protos`... ===> Running recipe `build-go-binary`... just build ftl 2>&1 6.32s user 3.05s system 95% cpu 9.799 total grep Running 0.00s user 0.00s system 0% cpu 9.798 total ``` After: ``` $ time just build-zips just build-zips 0.31s user 0.22s system 107% cpu 0.498 total $ time just go2proto just go2proto 0.04s user 0.02s system 20% cpu 0.277 total $ time just build ftl 2>&1 | grep Running ===> Running recipe `pnpm-install`... ===> Running recipe `build-frontend`... ===> Running recipe `build`... ===> Running recipe `go2proto`... ===> Running recipe `build-protos`... ===> Running recipe `build-zips`... ===> Running recipe `build-without-frontend`... ===> Running recipe `_build-go-binary-fast`... just build ftl 2>&1 3.59s user 2.62s system 159% cpu 3.903 total grep Running 0.00s user 0.00s system 0% cpu 3.902 total ``` --- .github/actions/build-cache/action.yml | 3 +++ Justfile | 35 +++++++++++++++++--------- bin/{.mk-0.9.1.pkg => .mk-0.10.0.pkg} | 0 bin/mk | 2 +- 4 files changed, 27 insertions(+), 13 deletions(-) rename bin/{.mk-0.9.1.pkg => .mk-0.10.0.pkg} (100%) diff --git a/.github/actions/build-cache/action.yml b/.github/actions/build-cache/action.yml index a1c46e027c..6c964e2291 100644 --- a/.github/actions/build-cache/action.yml +++ b/.github/actions/build-cache/action.yml @@ -4,6 +4,9 @@ runs: using: "composite" # Make sure to keep these cache entries in sync with those in writecache.yml steps: + - id: reset-timestamps + shell: bash + run: git ls-files -z | xargs -0 touch -r go.mod - id: find-go-build-cache shell: bash run: echo "cache=$(go env GOCACHE)" >> $GITHUB_OUTPUT diff --git a/Justfile b/Justfile index 20ece30847..830cf9167b 100644 --- a/Justfile +++ b/Justfile @@ -98,7 +98,7 @@ build-go2proto-testdata: @mk cmd/go2proto/testdata/testdatapb/model.pb.go : cmd/go2proto/testdata/testdatapb/model.proto -- '(cd ./cmd/go2proto/testdata/testdatapb && protoc --go_out=paths=source_relative:. model.proto) && go build ./cmd/go2proto/testdata' # Build command-line tools -build +tools: build-protos build-zips build-frontend +build +tools: build-frontend @just build-without-frontend $@ # Build command-line tools @@ -116,7 +116,7 @@ build-without-frontend +tools: build-protos build-zips for tool in $@; do path="cmd/$tool" test "$tool" = "ftl" && path="frontend/cli" - just build-go-binary "./$path" "$tool" + just _build-go-binary-fast "./$path" "$tool" done # Build all backend binaries @@ -132,14 +132,18 @@ build-jvm *args: mvn -f jvm-runtime/ftl-runtime install {{args}} # Builds all language plugins -build-language-plugins: - @just build-go-binary ./go-runtime/cmd/ftl-language-go - @just build-go-binary ./python-runtime/cmd/ftl-language-python - @just build-go-binary ./jvm-runtime/cmd/ftl-language-java - @just build-go-binary ./jvm-runtime/cmd/ftl-language-kotlin +build-language-plugins: build-zips build-protos + @just _build-go-binary-fast ./go-runtime/cmd/ftl-language-go ftl-language-go + @just _build-go-binary-fast ./python-runtime/cmd/ftl-language-python ftl-language-python + @just _build-go-binary-fast ./jvm-runtime/cmd/ftl-language-java ftl-language-java + @just _build-go-binary-fast ./jvm-runtime/cmd/ftl-language-kotlin ftl-language-kotlin # Build a Go binary with the correct flags and place it in the release dir build-go-binary dir binary="": build-zips build-protos + @just _build-go-binary-fast {{dir}} {{binary}} + +# Build Go binaries without first building zips/protos +_build-go-binary-fast dir binary="": #!/bin/bash set -euo pipefail shopt -s extglob @@ -154,7 +158,11 @@ build-go-binary dir binary="": build-zips build-protos # Build the ZIP files that are embedded in the FTL release binaries build-zips: - @for dir in {{ZIP_DIRS}}; do (cd $dir && mk ../$(basename ${dir}).zip : . -- "rm -f $(basename ${dir}.zip) && zip -q --symlinks -r ../$(basename ${dir}).zip ."); done + @echo {{ZIP_DIRS}} | xargs -P0 -n1 just _build-zip + +# This is separated due to command-length limits with xargs... +_build-zip dir: + @mk -C {{dir}} "../$(basename "{{dir}}").zip" : . -- "rm -f ../$(basename "{{dir}}").zip && zip -q --symlinks -r ../$(basename "{{dir}}").zip ." # Rebuild frontend build-frontend: pnpm-install @@ -197,7 +205,9 @@ format-frontend: # Install Node dependencies using pnpm pnpm-install: - @for i in {1..3}; do mk frontend/**/node_modules : frontend/**/package.json -- "pnpm install --frozen-lockfile" && break || sleep 5; done + #!/bin/bash + set -euo pipefail + for i in {1..3}; do mk frontend/{console,vscode}/node_modules : frontend/{console,vscode}/package.json -- "pnpm install --frozen-lockfile" && break || sleep 5; done # Copy plugin protos from the SQLC release update-sqlc-plugin-codegen-proto: @@ -209,10 +219,11 @@ build-protos: go2proto # Generate .proto files from .go types. go2proto: - @mk "{{SCHEMA_OUT}}" common/schema/go2proto.to.go : cmd/go2proto common/schema -- go2proto -m -o "{{SCHEMA_OUT}}" \ - -O 'go_package="github.com/block/ftl/common/protos/xyz/block/ftl/schema/v1;schemapb"' \ + @mk "{{SCHEMA_OUT}}" common/schema/go2proto.to.go : cmd/go2proto common/schema -- \ + "go2proto -m -o \"{{SCHEMA_OUT}}\" \ + -O 'go_package=\"github.com/block/ftl/common/protos/xyz/block/ftl/schema/v1;schemapb\"' \ -O 'java_multiple_files=true' \ - xyz.block.ftl.schema.v1 {{GO_SCHEMA_ROOTS}} && buf format -w && buf lint && bin/gofmt -w common/schema/go2proto.to.go + xyz.block.ftl.schema.v1 {{GO_SCHEMA_ROOTS}} && buf format -w && buf lint && bin/gofmt -w common/schema/go2proto.to.go" # Unconditionally rebuild protos build-protos-unconditionally: go2proto lint-protos pnpm-install diff --git a/bin/.mk-0.9.1.pkg b/bin/.mk-0.10.0.pkg similarity index 100% rename from bin/.mk-0.9.1.pkg rename to bin/.mk-0.10.0.pkg diff --git a/bin/mk b/bin/mk index f5306b7d04..cac1cecffd 120000 --- a/bin/mk +++ b/bin/mk @@ -1 +1 @@ -.mk-0.9.1.pkg \ No newline at end of file +.mk-0.10.0.pkg \ No newline at end of file