Merge branch 'main' into jonathanj/otel/pubsubtest-race-condition

.github/actions/build-cache/action.yml

@@ -31,5 +31,14 @@ runs:
         restore-keys: |
           ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
           ${{ runner.os }}-maven-
+    - name: Restore NPM Cache
+      id: cache-npm
+      uses: actions/cache/restore@v4
+      with:
+        path: .hermit/node/cache/_cacache
+        key: ${{ runner.os }}-npm-${{ hashFiles('**/package-lock.json') }}
+        restore-keys: |
+          ${{ runner.os }}-npm-${{ hashFiles('**/package-lock.json') }}
+          ${{ runner.os }}-npm-
 
 

.github/workflows/ci.yml

@@ -89,6 +89,8 @@ jobs:
         uses: ./.github/actions/build-cache
       - name: golangci-lint
         run: golangci-lint run --new-from-rev="$(git merge-base origin/main HEAD)" --out-format github-actions ./...
+      - name: lint-commit-or-rollback
+        run: lint-commit-or-rollback ./backend/... 2>&1 | to-annotation
       - name: go-check-sumtype
         shell: bash
         run: go-check-sumtype ./... 2>&1 | to-annotation
@@ -177,7 +179,7 @@ jobs:
       - run: cd docs && zola build
   integration-shard:
     name: Shard Integration Tests
-    if: github.event_name != 'pull_request' || github.event.action == 'enqueued'
+    if: github.event_name != 'pull_request' || github.event.action == 'enqueued' || contains( github.event.pull_request.labels.*.name, 'run-integration')
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.extract-tests.outputs.matrix }}
@@ -193,7 +195,7 @@ jobs:
           echo "matrix={\"test\":$(jq -c -n '$ARGS.positional' --args $(git grep -l '^//go:build integration' | xargs grep '^func Test' | awk '{print $2}' | cut -d'(' -f1))}" >> "$GITHUB_OUTPUT"
   integration-run:
     name: Integration Test
-    if: github.event_name != 'pull_request' || github.event.action == 'enqueued'
+    if: github.event_name != 'pull_request' || github.event.action == 'enqueued' || contains( github.event.pull_request.labels.*.name, 'run-integration')
     needs: integration-shard
     runs-on: ubuntu-latest
     strategy:

.github/workflows/writecache.yml

@@ -23,10 +23,13 @@ jobs:
         run: just build-all
       - name: Download Go Dependencies
         run: go mod download -x
+      - name: Download NPM Dependencies
+        run: find . -name package-lock.json -execdir npm ci \;
       - id: find-go-build-cache
         shell: bash
         run: echo "cache=$(go env GOCACHE)" >> "$GITHUB_OUTPUT"
-      - uses: actions/cache/save@v4
+      - name: Save Go Module Cache
+        uses: actions/cache/save@v4
         with:
           path: |
             ~/go/pkg/mod
@@ -38,3 +41,9 @@ jobs:
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+      - name: Save NPM Modules Cache
+        id: cache-npm
+        uses: actions/cache/save@v4
+        with:
+          path: .hermit/node/cache/_cacache
+          key: ${{ runner.os }}-npm-${{ hashFiles('**/package-lock.json') }}

CONTRIBUTING.md

@@ -12,9 +12,11 @@ This guide is for you.
 ## Development Prerequisites
 
 We recommend that you use OrbStack instead of Docker desktop when developing on this project:
+
 ```
 brew install orbstack
 ```
+
 or [OrbStack Website](https://orbstack.dev/)
 
 The tools used by this project are managed by
@@ -56,9 +58,18 @@ $ ftl dev --recreate ./examples/go
 
 ### Code reviews
 
-Because we're a geographically distributed team, we use a review-after-merge development flow. That is, if a PR is urgent, minor, or the developer has high confidence, we encourage merging without waiting for review in order to decrease friction. Conversely, if a change is more complex, or needs more eyes, we encourage developers to wait for review if it will make them feel more comfortable. Use your best judgement.
+Our goal is to maintain velocity while keeping code quality high. To that end, the process is an exercise in trust on the part of the reviewer, and responsibility on the part of the PR author.
+
+In practice, the **reviewer** will review _and_ approve the PR at the same time, trusting that the author will apply the feedback before merging.
+
+On the **author's** side, they are responsible for reading and understanding all feedback, and applying that feedback where they think it is appropriate. If either side doesn't understand something and it's important, comment accordingly, or do a quick pairing to resolve it. The author should feel free to re-request a review.
+
+Additional points of note:
 
-We discourage bike-shedding. Code and documentation are easy to change, we can always adjust it later.
+- We discourage bike-shedding. Code and documentation are easy to change, we can always adjust it later.
+- Keep your PRs digestible, large PRs are very difficult to comprehend and review.
+- Changing code is cheap, we can fix it later. The only caveat here is data storage.
+- Reviewing code is everybody's responsibility.
 
 ### Design process
 
@@ -135,6 +146,7 @@ just build-sqlc
 ```
 
 We use [dbmate](https://github.com/amacneil/dbmate) to manage migrations. To create a migration file, run `dbmate new` with the name of your migration. Example:
+
 ```
 dbmate new create_users_table
 ```
@@ -191,15 +203,19 @@ For an in-line replacement of `ftl dev <args>`, use the command:
 just debug <args>
 ```
 
-This command compiles a binary with debug information, runs `ftl dev <args>` using this binary, and provides an endpoint to attach a remote debugger at __127.0.0.1:2345__.
+This command compiles a binary with debug information, runs `ftl dev <args>` using this binary, and provides an endpoint to attach a remote debugger at **127.0.0.1:2345**.
 You do not need to run `FTL_DEBUG=true just build ftl` separately when using this command.
+
 ### Attaching a Debugger
 
 By running `just debug <args>` and then attaching a remote debugger, you can debug the FTL infrastructure while running your project.
 
 #### IntelliJ
+
 Run `Debug FTL` from the `Run/Debug Configurations` dropdown while in the FTL project.
+
 #### VSCode
+
 Run `Debug FTL` from the `Run and Debug` dropdown while in the FTL project.
 
 ## Useful links

Justfile

@@ -31,6 +31,7 @@ clean:
   rm -rf frontend/node_modules
   find . -name '*.zip' -exec rm {} \;
   mvn -f kotlin-runtime/ftl-runtime clean
+  mvn -f java-runtime/ftl-runtime clean
 
 # Live rebuild the ftl binary whenever source changes.
 live-rebuild:
@@ -41,7 +42,7 @@ dev *args:
   watchexec -r {{WATCHEXEC_ARGS}} -- "just build-sqlc && ftl dev {{args}}"
 
 # Build everything
-build-all: build-protos-unconditionally build-frontend build-generate build-sqlc build-zips lsp-generate
+build-all: build-protos-unconditionally build-frontend build-generate build-sqlc build-zips lsp-generate build-java
   @just build ftl ftl-controller ftl-runner ftl-initdb
 
 # Run "go generate" on all packages
@@ -64,6 +65,9 @@ build +tools: build-protos build-zips build-frontend
 build-backend:
   just build ftl ftl-controller ftl-runner
 
+build-java:
+  mvn -f java-runtime/ftl-runtime install
+
 export DATABASE_URL := "postgres://postgres:secret@localhost:15432/ftl?sslmode=disable"
 
 # Explicitly initialise the database
@@ -107,9 +111,10 @@ build-kt-runtime:
   @cd build/template && zip -q --symlinks -r ../../{{RUNNER_TEMPLATE_ZIP}} .
 
 # Install Node dependencies
+# npm install fails intermittently due to network issues, so we retry a few times.
 npm-install:
-  @mk frontend/node_modules : frontend/package.json -- "cd frontend && npm install"
-  @mk extensions/vscode/node_modules : extensions/vscode/package.json extensions/vscode/src -- "cd extensions/vscode && npm install"
+  @mk frontend/node_modules : frontend/package.json -- "cd frontend && for i in {1..3}; do npm install && break || sleep 5; done"
+  @mk extensions/vscode/node_modules : extensions/vscode/package.json extensions/vscode/src -- "cd extensions/vscode && for i in {1..3}; do npm install && break || sleep 5; done"
 
 # Regenerate protos
 build-protos: npm-install
@@ -158,6 +163,7 @@ lint-frontend: build-frontend
 # Lint the backend
 lint-backend:
   @golangci-lint run --new-from-rev=$(git merge-base origin/main HEAD) ./...
+  @lint-commit-or-rollback ./backend/...
 
 lint-scripts:
 	@shellcheck -f gcc -e SC2016 $(find scripts -type f -not -path scripts/tests) | to-annotation

backend/controller/admin/local_client_test.go

@@ -6,15 +6,18 @@ import (
 	"context"
 	"testing"
 
+	"github.com/alecthomas/assert/v2"
+	"github.com/alecthomas/types/optional"
+
 	cf "github.com/TBD54566975/ftl/common/configuration"
 	in "github.com/TBD54566975/ftl/integration"
 	"github.com/TBD54566975/ftl/internal/log"
-	"github.com/alecthomas/assert/v2"
-	"github.com/alecthomas/types/optional"
 )
 
 func TestDiskSchemaRetrieverWithBuildArtefact(t *testing.T) {
-	in.RunWithoutController(t, "ftl-project-dr.toml",
+	in.Run(t,
+		in.WithFTLConfig("ftl-project-dr.toml"),
+		in.WithoutController(),
 		in.CopyModule("dischema"),
 		in.Build("dischema"),
 		func(t testing.TB, ic in.TestContext) {
@@ -30,7 +33,9 @@ func TestDiskSchemaRetrieverWithBuildArtefact(t *testing.T) {
 }
 
 func TestDiskSchemaRetrieverWithNoSchema(t *testing.T) {
-	in.RunWithoutController(t, "ftl-project-dr.toml",
+	in.Run(t,
+		in.WithFTLConfig("ftl-project-dr.toml"),
+		in.WithoutController(),
 		in.CopyModule("dischema"),
 		func(t testing.TB, ic in.TestContext) {
 			dsr := &diskSchemaRetriever{}

backend/controller/console/console.go

@@ -195,7 +195,7 @@ func (c *ConsoleService) GetEvents(ctx context.Context, req *connect.Request[pbc
 	// Get 1 more than the requested limit to determine if there are more results.
 	limitPlusOne := limit + 1
 
-	results, err := c.dal.QueryEvents(ctx, limitPlusOne, query...)
+	results, err := c.dal.QueryTimeline(ctx, limitPlusOne, query...)
 	if err != nil {
 		return nil, err
 	}
@@ -241,7 +241,7 @@ func (c *ConsoleService) StreamEvents(ctx context.Context, req *connect.Request[
 			newQuery = append(newQuery, dal.FilterTimeRange(thisRequestTime, lastEventTime))
 		}
 
-		events, err := c.dal.QueryEvents(ctx, int(req.Msg.Query.Limit), newQuery...)
+		events, err := c.dal.QueryTimeline(ctx, int(req.Msg.Query.Limit), newQuery...)
 		if err != nil {
 			return err
 		}
@@ -264,8 +264,8 @@ func (c *ConsoleService) StreamEvents(ctx context.Context, req *connect.Request[
 	}
 }
 
-func eventsQueryProtoToDAL(pb *pbconsole.EventsQuery) ([]dal.EventFilter, error) {
-	var query []dal.EventFilter
+func eventsQueryProtoToDAL(pb *pbconsole.EventsQuery) ([]dal.TimelineFilter, error) {
+	var query []dal.TimelineFilter
 
 	if pb.Order == pbconsole.EventsQuery_DESC {
 		query = append(query, dal.FilterDescending())
@@ -357,7 +357,7 @@ func eventsQueryProtoToDAL(pb *pbconsole.EventsQuery) ([]dal.EventFilter, error)
 	return query, nil
 }
 
-func eventDALToProto(event dal.Event) *pbconsole.Event {
+func eventDALToProto(event dal.TimelineEvent) *pbconsole.Event {
 	switch event := event.(type) {
 	case *dal.CallEvent:
 		var requestKey *string

backend/controller/console/console_integration_test.go

@@ -6,9 +6,10 @@ import (
 	"testing"
 
 	"connectrpc.com/connect"
+	"github.com/alecthomas/assert/v2"
+
 	pbconsole "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/console"
 	in "github.com/TBD54566975/ftl/integration"
-	"github.com/alecthomas/assert/v2"
 )
 
 // GetModules calls console service GetModules and returns the response.
@@ -24,7 +25,7 @@ func GetModules(onResponse func(t testing.TB, resp *connect.Response[pbconsole.G
 }
 
 func TestConsoleGetModules(t *testing.T) {
-	in.Run(t, "",
+	in.Run(t,
 		in.CopyModule("console"),
 		in.Deploy("console"),
 		GetModules(func(t testing.TB, resp *connect.Response[pbconsole.GetModulesResponse]) {

backend/controller/console/testdata/go/console/go.mod

@@ -7,6 +7,7 @@ require github.com/TBD54566975/ftl v0.189.0
 require (
 	connectrpc.com/connect v1.16.2 // indirect
 	connectrpc.com/grpcreflect v1.2.0 // indirect
+	connectrpc.com/otelconnect v0.7.1 // indirect
 	github.com/alecthomas/atomic v0.1.0-alpha2 // indirect
 	github.com/alecthomas/concurrency v0.0.2 // indirect
 	github.com/alecthomas/participle/v2 v2.1.1 // indirect
@@ -33,13 +34,13 @@ require (
 	go.opentelemetry.io/otel v1.28.0 // indirect
 	go.opentelemetry.io/otel/metric v1.28.0 // indirect
 	go.opentelemetry.io/otel/trace v1.28.0 // indirect
-	golang.org/x/crypto v0.25.0 // indirect
+	golang.org/x/crypto v0.26.0 // indirect
 	golang.org/x/exp v0.0.0-20240707233637-46b078467d37 // indirect
-	golang.org/x/mod v0.19.0 // indirect
-	golang.org/x/net v0.27.0 // indirect
-	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/sys v0.22.0 // indirect
-	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/mod v0.20.0 // indirect
+	golang.org/x/net v0.28.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sys v0.24.0 // indirect
+	golang.org/x/text v0.17.0 // indirect
 	google.golang.org/protobuf v1.34.2 // indirect
 )