feat: language plugins own build locks (#3151)

Previously the build engine would obtain a file lock before building
each module. This helps protect against mutiple FTL build engines trying
to build the same module at the same time (eg `ftl dev` and `ftl
build`).

With language plugins, each plugin is responsible for detecting file
changes and kicking off the a new build without first coordinating with
the central build engine. Therefore build locks have to be coordinated
by the language plugins.

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

backend/protos/xyz/block/ftl/v1/language/language.proto

@@ -22,14 +22,16 @@ message ModuleConfig {
   string deploy_dir = 4;
   // Build is the command to build the module.
   optional string build = 5;
+  // Build lock path to prevent concurrent builds
+  string build_lock = 6;
   // The directory to generate protobuf schema files into. These can be picked up by language specific build tools
-  optional string generated_schema_dir = 6;
+  optional string generated_schema_dir = 7;
   // Patterns to watch for file changes
-  repeated string watch = 7;
+  repeated string watch = 8;
 
   // LanguageConfig contains any metadata specific to a specific language.
   // These are stored in the ftl.toml file under the same key as the language (eg: "go", "java")
-  google.protobuf.Struct language_config = 8;
+  google.protobuf.Struct language_config = 9;
 }
 
 // ProjectConfig contains the configuration for a project, found in the ftl-project.toml file.
@@ -92,15 +94,18 @@ message ModuleConfigDefaultsResponse {
   // Default build command
   optional string build = 2;
 
+  // Build lock path to prevent concurrent builds
+  optional string build_lock = 3;
+
   // Default relative path to the directory containing generated schema files
-  optional string generated_schema_dir = 3;
+  optional string generated_schema_dir = 4;
 
   // Default patterns to watch for file changes, relative to the module directory
-  repeated string watch = 4;
+  repeated string watch = 5;
 
   // Default language specific configuration.
   // These defaults are filled in by looking at each root key only. If the key is not present, the default is used.
-  google.protobuf.Struct language_config = 5;
+  google.protobuf.Struct language_config = 6;
 }
 
 message DependenciesRequest {

backend/protos/xyz/block/ftl/v1/language/mixins.go

@@ -119,6 +119,7 @@ func ModuleConfigToProto(config moduleconfig.AbsModuleConfig) (*ModuleConfig, er
 		Name:      config.Module,
 		Dir:       config.Dir,
 		DeployDir: config.DeployDir,
+		BuildLock: config.BuildLock,
 		Watch:     config.Watch,
 		Language:  config.Language,
 	}
@@ -147,6 +148,7 @@ func ModuleConfigFromProto(proto *ModuleConfig) moduleconfig.AbsModuleConfig {
 		Watch:              proto.Watch,
 		Language:           proto.Language,
 		Build:              proto.GetBuild(),
+		BuildLock:          proto.BuildLock,
 		GeneratedSchemaDir: proto.GetGeneratedSchemaDir(),
 	}
 	if proto.LanguageConfig != nil {

internal/buildengine/build.go

@@ -21,7 +21,7 @@ var errInvalidateDependencies = errors.New("dependencies need to be updated")
 
 // Build a module in the given directory given the schema and module config.
 //
-// A lock file is used to ensure that only one build is running at a time.
+// Plugins must use a lock file to ensure that only one build is running at a time.
 //
 // Returns invalidateDependenciesError if the build failed due to a change in dependencies.
 func build(ctx context.Context, plugin languageplugin.LanguagePlugin, projectRootDir string, bctx languageplugin.BuildContext, buildEnv []string, devMode bool) (moduleSchema *schema.Module, deploy []string, err error) {

internal/buildengine/languageplugin/external_plugin.go

@@ -162,6 +162,7 @@ func customDefaultsFromProto(proto *langpb.ModuleConfigDefaultsResponse) modulec
 		DeployDir:          proto.DeployDir,
 		Watch:              proto.Watch,
 		Build:              optional.Ptr(proto.Build),
+		BuildLock:          optional.Ptr(proto.BuildLock),
 		GeneratedSchemaDir: optional.Ptr(proto.GeneratedSchemaDir),
 		LanguageConfig:     proto.LanguageConfig.AsMap(),
 	}

internal/buildengine/languageplugin/plugin.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"os"
-	"path/filepath"
 	"time"
 
 	"github.com/alecthomas/kong"
@@ -339,7 +338,7 @@ func (p *internalPlugin) run(ctx context.Context) {
 
 func buildAndLoadResult(ctx context.Context, projectRoot, stubsRoot string, bctx BuildContext, buildEnv []string, devMode bool, watcher *watch.Watcher, build buildFunc) (BuildResult, error) {
 	config := bctx.Config.Abs()
-	release, err := flock.Acquire(ctx, filepath.Join(config.Dir, ".ftl.lock"), BuildLockTimeout)
+	release, err := flock.Acquire(ctx, config.BuildLock, BuildLockTimeout)
 	if err != nil {
 		return BuildResult{}, fmt.Errorf("could not acquire build lock for %v: %w", config.Module, err)
 	}

internal/moduleconfig/moduleconfig.go

@@ -24,6 +24,8 @@ type ModuleConfig struct {
 	Module   string `toml:"module"`
 	// Build is the command to build the module.
 	Build string `toml:"build"`
+	// BuildLock is file lock path to prevent concurrent builds of a module.
+	BuildLock string `toml:"build-lock"`
 	// DeployDir is the directory to deploy from, relative to the module directory.
 	DeployDir string `toml:"deploy-dir"`
 	// GeneratedSchemaDir is the directory to generate protobuf schema files into. These can be picked up by language specific build tools
@@ -54,6 +56,7 @@ type UnvalidatedModuleConfig ModuleConfig
 type CustomDefaults struct {
 	DeployDir          string
 	Watch              []string
+	BuildLock          optional.Option[string]
 	Build              optional.Option[string]
 	GeneratedSchemaDir optional.Option[string]
 
@@ -117,6 +120,7 @@ func (c ModuleConfig) Abs() AbsModuleConfig {
 	if !strings.HasPrefix(clone.DeployDir, clone.Dir) {
 		panic(fmt.Sprintf("deploy-dir %q is not beneath module directory %q", clone.DeployDir, clone.Dir))
 	}
+	clone.BuildLock = filepath.Clean(filepath.Join(clone.Dir, clone.BuildLock))
 	if clone.GeneratedSchemaDir != "" {
 		clone.GeneratedSchemaDir = filepath.Clean(filepath.Join(clone.Dir, clone.GeneratedSchemaDir))
 		if !strings.HasPrefix(clone.GeneratedSchemaDir, clone.Dir) {
@@ -142,6 +146,13 @@ func (c UnvalidatedModuleConfig) FillDefaultsAndValidate(customDefaults CustomDe
 	if defaultValue, ok := customDefaults.Build.Get(); ok && c.Build == "" {
 		c.Build = defaultValue
 	}
+	if c.BuildLock == "" {
+		if defaultValue, ok := customDefaults.BuildLock.Get(); ok {
+			c.BuildLock = defaultValue
+		} else {
+			c.BuildLock = ".ftl.lock"
+		}
+	}
 	if c.DeployDir == "" {
 		c.DeployDir = customDefaults.DeployDir
 	}
@@ -166,6 +177,9 @@ func (c UnvalidatedModuleConfig) FillDefaultsAndValidate(customDefaults CustomDe
 	if c.DeployDir == "" {
 		return ModuleConfig{}, fmt.Errorf("no deploy directory configured")
 	}
+	if c.BuildLock == "" {
+		return ModuleConfig{}, fmt.Errorf("no build lock path configured")
+	}
 	if !isBeneath(c.Dir, c.DeployDir) {
 		return ModuleConfig{}, fmt.Errorf("deploy-dir %s must be relative to the module directory %s", c.DeployDir, c.Dir)
 	}

internal/moduleconfig/moduleconfig_test.go

@@ -23,6 +23,7 @@ func TestDefaulting(t *testing.T) {
 			},
 			defaults: CustomDefaults{
 				Build:              optional.Some("build"),
+				BuildLock:          optional.Some("customdefaultlock"),
 				DeployDir:          "deploydir",
 				GeneratedSchemaDir: optional.Some("generatedschemadir"),
 				Watch:              []string{"a", "b", "c"},
@@ -33,6 +34,7 @@ func TestDefaulting(t *testing.T) {
 				Module:             "nothingset",
 				Language:           "test",
 				Build:              "build",
+				BuildLock:          "customdefaultlock",
 				DeployDir:          "deploydir",
 				GeneratedSchemaDir: "generatedschemadir",
 				Watch:              []string{"a", "b", "c"},
@@ -44,6 +46,7 @@ func TestDefaulting(t *testing.T) {
 				Module:             "allset",
 				Language:           "test",
 				Build:              "custombuild",
+				BuildLock:          "custombuildlock",
 				DeployDir:          "customdeploydir",
 				GeneratedSchemaDir: "customgeneratedschemadir",
 				Watch:              []string{"custom1"},
@@ -64,6 +67,7 @@ func TestDefaulting(t *testing.T) {
 				Module:             "allset",
 				Language:           "test",
 				Build:              "custombuild",
+				BuildLock:          "custombuildlock",
 				DeployDir:          "customdeploydir",
 				GeneratedSchemaDir: "customgeneratedschemadir",
 				Watch:              []string{"custom1"},
@@ -103,6 +107,7 @@ func TestDefaulting(t *testing.T) {
 				Realm:     "home",
 				Dir:       "b",
 				Module:    "languageconfig",
+				BuildLock: ".ftl.lock",
 				Language:  "test",
 				LanguageConfig: map[string]any{
 					"alreadyset": "correct",