Skip to content

Commit

Permalink
chore: speed up parts of the build (#3812)
Browse files Browse the repository at this point in the history
This should have a fairly decent cumulative effect.

Before:

```
$ time just build-zips
just build-zips  0.20s user 0.09s system 22% cpu 1.287 total
$ time just go2proto
just go2proto  0.83s user 0.09s system 84% cpu 1.090 total
$ time just build ftl 2>&1 | grep Running
===> Running recipe `go2proto`...
===> Running recipe `build-protos`...
===> Running recipe `build-zips`...
===> Running recipe `pnpm-install`...
===> Running recipe `build-frontend`...
===> Running recipe `build`...
===> Running recipe `go2proto`...
===> Running recipe `build-protos`...
===> Running recipe `build-zips`...
===> Running recipe `build-without-frontend`...
===> Running recipe `build-zips`...
===> Running recipe `go2proto`...
===> Running recipe `build-protos`...
===> Running recipe `build-go-binary`...
just build ftl 2>&1  6.32s user 3.05s system 95% cpu 9.799 total
grep Running  0.00s user 0.00s system 0% cpu 9.798 total
```

After:

```
$ time just build-zips
just build-zips  0.31s user 0.22s system 107% cpu 0.498 total
$ time just go2proto
just go2proto  0.04s user 0.02s system 20% cpu 0.277 total
$ time just build ftl 2>&1 | grep Running
===> Running recipe `pnpm-install`...
===> Running recipe `build-frontend`...
===> Running recipe `build`...
===> Running recipe `go2proto`...
===> Running recipe `build-protos`...
===> Running recipe `build-zips`...
===> Running recipe `build-without-frontend`...
===> Running recipe `_build-go-binary-fast`...
just build ftl 2>&1  3.59s user 2.62s system 159% cpu 3.903 total
grep Running  0.00s user 0.00s system 0% cpu 3.902 total
```
  • Loading branch information
alecthomas authored Dec 18, 2024
1 parent ffbf1d6 commit 004a8e8
Show file tree
Hide file tree
Showing 4 changed files with 27 additions and 13 deletions.
3 changes: 3 additions & 0 deletions .github/actions/build-cache/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ runs:
using: "composite"
# Make sure to keep these cache entries in sync with those in writecache.yml
steps:
- id: reset-timestamps
shell: bash
run: git ls-files -z | xargs -0 touch -r go.mod
- id: find-go-build-cache
shell: bash
run: echo "cache=$(go env GOCACHE)" >> $GITHUB_OUTPUT
Expand Down
35 changes: 23 additions & 12 deletions Justfile
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ build-go2proto-testdata:
@mk cmd/go2proto/testdata/testdatapb/model.pb.go : cmd/go2proto/testdata/testdatapb/model.proto -- '(cd ./cmd/go2proto/testdata/testdatapb && protoc --go_out=paths=source_relative:. model.proto) && go build ./cmd/go2proto/testdata'

# Build command-line tools
build +tools: build-protos build-zips build-frontend
build +tools: build-frontend
@just build-without-frontend $@

# Build command-line tools
Expand All @@ -116,7 +116,7 @@ build-without-frontend +tools: build-protos build-zips
for tool in $@; do
path="cmd/$tool"
test "$tool" = "ftl" && path="frontend/cli"
just build-go-binary "./$path" "$tool"
just _build-go-binary-fast "./$path" "$tool"
done

# Build all backend binaries
Expand All @@ -132,14 +132,18 @@ build-jvm *args:
mvn -f jvm-runtime/ftl-runtime install {{args}}

# Builds all language plugins
build-language-plugins:
@just build-go-binary ./go-runtime/cmd/ftl-language-go
@just build-go-binary ./python-runtime/cmd/ftl-language-python
@just build-go-binary ./jvm-runtime/cmd/ftl-language-java
@just build-go-binary ./jvm-runtime/cmd/ftl-language-kotlin
build-language-plugins: build-zips build-protos
@just _build-go-binary-fast ./go-runtime/cmd/ftl-language-go ftl-language-go
@just _build-go-binary-fast ./python-runtime/cmd/ftl-language-python ftl-language-python
@just _build-go-binary-fast ./jvm-runtime/cmd/ftl-language-java ftl-language-java
@just _build-go-binary-fast ./jvm-runtime/cmd/ftl-language-kotlin ftl-language-kotlin

# Build a Go binary with the correct flags and place it in the release dir
build-go-binary dir binary="": build-zips build-protos
@just _build-go-binary-fast {{dir}} {{binary}}

# Build Go binaries without first building zips/protos
_build-go-binary-fast dir binary="":
#!/bin/bash
set -euo pipefail
shopt -s extglob
Expand All @@ -154,7 +158,11 @@ build-go-binary dir binary="": build-zips build-protos

# Build the ZIP files that are embedded in the FTL release binaries
build-zips:
@for dir in {{ZIP_DIRS}}; do (cd $dir && mk ../$(basename ${dir}).zip : . -- "rm -f $(basename ${dir}.zip) && zip -q --symlinks -r ../$(basename ${dir}).zip ."); done
@echo {{ZIP_DIRS}} | xargs -P0 -n1 just _build-zip

# This is separated due to command-length limits with xargs...
_build-zip dir:
@mk -C {{dir}} "../$(basename "{{dir}}").zip" : . -- "rm -f ../$(basename "{{dir}}").zip && zip -q --symlinks -r ../$(basename "{{dir}}").zip ."

# Rebuild frontend
build-frontend: pnpm-install
Expand Down Expand Up @@ -197,7 +205,9 @@ format-frontend:

# Install Node dependencies using pnpm
pnpm-install:
@for i in {1..3}; do mk frontend/**/node_modules : frontend/**/package.json -- "pnpm install --frozen-lockfile" && break || sleep 5; done
#!/bin/bash
set -euo pipefail
for i in {1..3}; do mk frontend/{console,vscode}/node_modules : frontend/{console,vscode}/package.json -- "pnpm install --frozen-lockfile" && break || sleep 5; done

# Copy plugin protos from the SQLC release
update-sqlc-plugin-codegen-proto:
Expand All @@ -209,10 +219,11 @@ build-protos: go2proto

# Generate .proto files from .go types.
go2proto:
@mk "{{SCHEMA_OUT}}" common/schema/go2proto.to.go : cmd/go2proto common/schema -- go2proto -m -o "{{SCHEMA_OUT}}" \
-O 'go_package="github.com/block/ftl/common/protos/xyz/block/ftl/schema/v1;schemapb"' \
@mk "{{SCHEMA_OUT}}" common/schema/go2proto.to.go : cmd/go2proto common/schema -- \
"go2proto -m -o \"{{SCHEMA_OUT}}\" \
-O 'go_package=\"github.com/block/ftl/common/protos/xyz/block/ftl/schema/v1;schemapb\"' \
-O 'java_multiple_files=true' \
xyz.block.ftl.schema.v1 {{GO_SCHEMA_ROOTS}} && buf format -w && buf lint && bin/gofmt -w common/schema/go2proto.to.go
xyz.block.ftl.schema.v1 {{GO_SCHEMA_ROOTS}} && buf format -w && buf lint && bin/gofmt -w common/schema/go2proto.to.go"

# Unconditionally rebuild protos
build-protos-unconditionally: go2proto lint-protos pnpm-install
Expand Down
File renamed without changes.
2 changes: 1 addition & 1 deletion bin/mk

0 comments on commit 004a8e8

Please sign in to comment.