chore: replace bit with Justfile (#1109)

This was way more work and way more irritating than I hoped it would be.
I was hoping to switch back to make, but make doesn't support spaces in
filenames. Then I tried ninja, which doesn't support "|" in filenames.
Finally, I tried just using Just, but the lack of a dependency system
meant that really slow targets like the kt-runtime and npm made it
unreasonably painful.

So in the end I wrote a [little
tool](https://github.com/alecthomas/mktg) which acts like a single
"make" target entry, executing the provided command if any input is
newer than all outputs. This gives us fairly fast incremental builds
with little complexity.

.github/workflows/integration.yml

@@ -37,6 +37,8 @@ jobs:
         uses: ./.github/actions/build-cache
       - name: Docker Compose
         run: docker compose up -d --wait
+      - name: Create DB
+        run: just init-db
       - name: Download Go Modules
         run: go mod download
       - name: Run ${{ matrix.test }}

.github/workflows/rebuild.yml

@@ -23,7 +23,5 @@ jobs:
         run: docker compose up -d --wait
       - name: Init DB
         run: ftl-initdb
-      - name: Clean All
-        run: bit -c
       - name: Rebuild All
-        run: bit
+        run: just build-all

.github/workflows/release.yml

@@ -134,11 +134,11 @@ jobs:
       - name: Build Cache
         uses: ./.github/actions/build-cache
       - name: Build Console
-        run: bit frontend/dist/index.html
+        run: just build-frontend
       - name: Publish Go Binaries
         run: |
           just errtrace
-          bit build/release/ftl
+          just build ftl # Ensure all the prerequisites are built before we use goreleaser
           goreleaser release --skip=validate
         env:
           GITHUB_TOKEN: ${{ github.token }}

.github/workflows/writecache.yml

@@ -19,10 +19,8 @@ jobs:
         run: docker compose up -d --wait
       - name: Init DB
         run: ftl-initdb
-      - name: Clean All
-        run: bit -c
       - name: Rebuild All
-        run: bit
+        run: bit build-all
       - name: Download Maven Dependencies
         run: mvn -f kotlin-runtime/ftl-runtime dependency:resolve --batch-mode
       - name: Download Go Dependencies

Dockerfile.controller

@@ -20,9 +20,9 @@ RUN go mod download -x
 # Build
 COPY . /src/
 RUN just errtrace
-RUN bit build/release/ftl-controller
-RUN bit build/release/ftl-initdb
-RUN bit build/release/ftl
+RUN just build ftl-controller
+RUN just build ftl-initdb
+RUN just build ftl
 
 # Finally create the runtime image.
 FROM ubuntu:22.04

Dockerfile.runner

@@ -23,12 +23,12 @@ RUN go mod download -x
 COPY . /src/
 
 # Build runner template
-RUN bit build/template/ftl/jars/ftl-runtime.jar
+RUN just build-kt-runtime
 
 # Build runner
 RUN just errtrace
-RUN bit build/release/ftl-runner
-RUN bit build/release/ftl
+RUN just build ftl-runner
+RUN just build ftl
 
 # Finally create the runtime image.
 FROM ubuntu:22.04

Justfile

@@ -1,25 +1,65 @@
 set positional-arguments
 
-# Start a hot-reloading dev cluster
-dev: install-jars
-  goreman -logtime=false start
+RELEASE := "build/release"
+VERSION := `git describe --tags --always --dirty | sed -e 's/^v//'`
+KT_RUNTIME_OUT := "kotlin-runtime/ftl-runtime/target/ftl-runtime-1.0-SNAPSHOT.jar"
+KT_RUNTIME_RUNNER_TEMPLATE_OUT := "build/template/ftl/jars/ftl-runtime.jar"
+RUNNER_TEMPLATE_ZIP := "backend/controller/scaling/localscaling/template.zip"
+TIMESTAMP := `date +%s`
+SCHEMA_OUT := "backend/protos/xyz/block/ftl/v1/schema/schema.proto"
+ZIP_DIRS := "go-runtime/compile/build-template go-runtime/compile/external-module-template go-runtime/scaffolding kotlin-runtime/scaffolding kotlin-runtime/external-module-template"
+FRONTEND_OUT := "frontend/dist/index.html"
 
-# Install all JARs to local Maven repository and local build directory
-install-jars:
-  bit 'kotlin-runtime/**/*.jar'
+_help:
+  @just -l
 
-# Deploy the Go time module
-deploy-time:
-  ftl deploy examples/time
+# Run errtrace on Go files to add stacks
+errtrace:
+  git ls-files -z -- '*.go' | grep -zv /_ | xargs -0 errtrace -w && go mod tidy
 
-# Deploy the Kotlin echo module
-deploy-echo-kotlin:
-  ftl deploy examples/echo-kotlin
+# Build everything
+build-all: build-frontend build-generate build-kt-runtime build-protos build-sqlc build-zips
+  just build ftl
 
-regen-schema:
-  bit backend/protos/xyz/block/ftl/v1/schema/schema.proto
-  bit backend/protos/xyz/block/ftl/v1/schema/schema.pb.go
+# Run "go generate" on all packages
+build-generate:
+  go generate -x ./...
 
-# Run errtrace on Go files to add stacks
-errtrace:
-  git ls-files -z -- '*.go' | grep -zv /_ | xargs -0 errtrace -w && go mod tidy
+# Build command-line tools
+build +tools: build-protos build-sqlc build-zips build-frontend
+  for tool in $@; do go build -o "{{RELEASE}}/$tool" -tags release -ldflags "-X github.com/TBD54566975/ftl.Version={{VERSION}} -X github.com/TBD54566975/ftl.Timestamp={{TIMESTAMP}}" "./cmd/$tool"; done
+
+export DATABASE_URL := "postgres://postgres:secret@localhost:54320/ftl?sslmode=disable"
+
+# Explicitly initialise the database
+init-db:
+  dbmate drop || true
+  dbmate create
+  dbmate --migrations-dir backend/controller/sql/schema up
+
+# Regenerate SQLC code
+build-sqlc:
+  sqlc generate --experimental
+
+# Build the ZIP files that are embedded in the FTL release binaries
+build-zips: build-kt-runtime
+  for dir in {{ZIP_DIRS}}; do (cd $dir && rm -f $(basename ${dir}.zip) && zip -q --symlinks -r ../$(basename ${dir}).zip .); done
+
+# Rebuild frontend
+build-frontend: npm-install
+  mktg {{FRONTEND_OUT}} : frontend/src -- "cd frontend && npm run build"
+
+# Build the Kotlin runtime (if necessary)
+build-kt-runtime:
+  mkdir -p $(dirname {{KT_RUNTIME_RUNNER_TEMPLATE_OUT}})
+  mktg {{KT_RUNTIME_OUT}} : kotlin-runtime/ftl-runtime -- mvn -f kotlin-runtime/ftl-runtime -Dmaven.test.skip=true -B install
+  install -m 0600 {{KT_RUNTIME_OUT}} {{KT_RUNTIME_RUNNER_TEMPLATE_OUT}}
+  mktg {{RUNNER_TEMPLATE_ZIP}} : {{KT_RUNTIME_RUNNER_TEMPLATE_OUT}} -- "cd build/template && zip -q --symlinks -r ../../{{RUNNER_TEMPLATE_ZIP}} ."
+
+# Install Node dependencies
+npm-install:
+  mktg frontend/node_modules : frontend/package.json frontend/src -- "cd frontend && npm install"
+
+# Regenerate protos
+build-protos: npm-install
+  mktg {{SCHEMA_OUT}} : backend/schema -- "ftl-schema > {{SCHEMA_OUT}} && buf format -w && buf lint && cd backend/protos && buf generate"

backend/controller/scaling/localscaling/devel.go

@@ -17,7 +17,7 @@ var templateDirOnce sync.Once
 func templateDir(ctx context.Context) string {
 	templateDirOnce.Do(func() {
 		// TODO: Figure out how to make maven build offline
-		err := exec.Command(ctx, log.Debug, internal.GitRoot(""), "bit", "build/template/ftl/jars/ftl-runtime.jar").RunBuffered(ctx)
+		err := exec.Command(ctx, log.Debug, internal.GitRoot(""), "just", "build-kt-runtime").RunBuffered(ctx)
 		if err != nil {
 			panic(err)
 		}

bin/.mktg-0.3.0.pkg

@@ -0,0 +1 @@
+hermit

bin/dbmate

@@ -0,0 +1 @@
+.dbmate-2.13.0.pkg

bin/mktg

@@ -0,0 +1 @@
+.mktg-0.3.0.pkg

frontend/local.go

@@ -22,7 +22,7 @@ func Server(ctx context.Context, timestamp time.Time, publicURL *url.URL, allowO
 	logger := log.FromContext(ctx)
 	logger.Debugf("Building console...")
 
-	err := exec.Command(ctx, log.Debug, internal.GitRoot(""), "bit", "frontend/**/*").RunBuffered(ctx)
+	err := exec.Command(ctx, log.Debug, internal.GitRoot(""), "just", "build-frontend").RunBuffered(ctx)
 	if err != nil {
 		return nil, err
 	}

go-runtime/release.go

@@ -12,9 +12,6 @@ import (
 var archive []byte
 
 // Files is the FTL Go runtime scaffolding files.
-//
-// scaffolding.zip can be generated by running `bit go-runtime/scaffolding.zip`
-// or indirectly via `bit build/release/ftl`.
 func Files() *zip.Reader {
 	zr, err := zip.NewReader(bytes.NewReader(archive), int64(len(archive)))
 	if err != nil {

integration/integration_test.go

@@ -272,7 +272,7 @@ func runTests(t *testing.T, tmpDir string, tests []test) {
 	ctx := log.ContextWithLogger(context.Background(), logger)
 	logger.Debugf("Building ftl")
 	binDir := filepath.Join(rootDir, "build", "release")
-	err = exec.Command(ctx, log.Debug, rootDir, filepath.Join(rootDir, "bin", "bit"), "build/release/ftl", "**/*.jar").RunBuffered(ctx)
+	err = exec.Command(ctx, log.Debug, rootDir, "just", "build", "ftl").RunBuffered(ctx)
 	assert.NoError(t, err)
 
 	controller := rpc.Dial(ftlv1connect.NewControllerServiceClient, "http://localhost:8892", log.Debug)

kotlin-runtime/release.go

@@ -15,9 +15,6 @@ var archive []byte
 var externalModuleTemplate []byte
 
 // Files is the FTL Kotlin runtime scaffolding files.
-//
-// scaffolding.zip can be generated by running `bit kotlin-runtime/scaffolding.zip`
-// or indirectly via `bit build/release/ftl`.
 func Files() *zip.Reader {
 	zr, err := zip.NewReader(bytes.NewReader(archive), int64(len(archive)))
 	if err != nil {
@@ -27,9 +24,6 @@ func Files() *zip.Reader {
 }
 
 // ExternalModuleTemplates are templates for scaffolding external modules in the FTL Kotlin runtime.
-//
-// external-module-template.zip can be generated by running `bit kotlin-runtime/external-module-template.zip`
-// or indirectly via `bit build/release/ftl`.
 func ExternalModuleTemplates() *zip.Reader {
 	zr, err := zip.NewReader(bytes.NewReader(externalModuleTemplate), int64(len(externalModuleTemplate)))
 	if err != nil {

scripts/ftl-run

@@ -6,9 +6,7 @@ cd "${top}"
 
 
 build() {
-  bit build/template/ftl/jars/ftl-runtime.jar \
-    build/release/ftl-controller \
-    build/release/ftl-runner 
+  just build ftl-controller ftl-runner
 }
 
 build