diff --git a/backend/controller/dal/testdata/go/fsm/go.sum b/backend/controller/dal/testdata/go/fsm/go.sum index be1709c712..5b05f397e5 100644 --- a/backend/controller/dal/testdata/go/fsm/go.sum +++ b/backend/controller/dal/testdata/go/fsm/go.sum @@ -74,8 +74,6 @@ github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpv github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= -github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= diff --git a/backend/controller/leases/testdata/go/leases/go.sum b/backend/controller/leases/testdata/go/leases/go.sum index be1709c712..5b05f397e5 100644 --- a/backend/controller/leases/testdata/go/leases/go.sum +++ b/backend/controller/leases/testdata/go/leases/go.sum @@ -74,8 +74,6 @@ github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpv github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= -github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= diff --git a/backend/controller/sql/sqltest/testing.go b/backend/controller/sql/sqltest/testing.go index 48345052aa..058b36134a 100644 --- a/backend/controller/sql/sqltest/testing.go +++ b/backend/controller/sql/sqltest/testing.go @@ -8,25 +8,21 @@ import ( "time" "github.com/alecthomas/assert/v2" - "github.com/gofrs/flock" "github.com/jackc/pgx/v5/pgxpool" "github.com/TBD54566975/ftl/backend/controller/sql/databasetesting" + "github.com/TBD54566975/ftl/internal/flock" ) // OpenForTesting opens a database connection for testing, recreating the // database beforehand. func OpenForTesting(ctx context.Context, t testing.TB) *pgxpool.Pool { t.Helper() - ctx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() // Acquire lock for this DB. lockPath := filepath.Join(os.TempDir(), "ftl-db-test.lock") - lock := flock.New(lockPath) - ok, err := lock.TryLockContext(ctx, time.Second) + release, err := flock.Acquire(ctx, lockPath, 10*time.Second) assert.NoError(t, err) - assert.True(t, ok, "could not acquire lock on %s", lockPath) - t.Cleanup(func() { _ = lock.Unlock() }) + t.Cleanup(func() { _ = release() }) testDSN := "postgres://localhost:54320/ftl-test?user=postgres&password=secret&sslmode=disable" conn, err := databasetesting.CreateForDevel(ctx, testDSN, true) diff --git a/backend/controller/sql/testdata/go/database/go.sum b/backend/controller/sql/testdata/go/database/go.sum index be1709c712..5b05f397e5 100644 --- a/backend/controller/sql/testdata/go/database/go.sum +++ b/backend/controller/sql/testdata/go/database/go.sum @@ -74,8 +74,6 @@ github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpv github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= -github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= diff --git a/buildengine/build.go b/buildengine/build.go index 0705f23302..a5049d18ba 100644 --- a/buildengine/build.go +++ b/buildengine/build.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "strings" + "time" "google.golang.org/protobuf/proto" @@ -13,12 +14,18 @@ import ( "github.com/TBD54566975/ftl/backend/schema" "github.com/TBD54566975/ftl/common/moduleconfig" "github.com/TBD54566975/ftl/internal/errors" + "github.com/TBD54566975/ftl/internal/flock" "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/slices" ) +const BuildLockTimeout = time.Minute + // Build a project in the given directory given the schema and project config. +// // For a module, this will build the module. For an external library, this will build stubs for imported modules. +// +// A lock file is used to ensure that only one build is running at a time. func Build(ctx context.Context, sch *schema.Schema, project Project, filesTransaction ModifyFilesTransaction) error { switch project := project.(type) { case Module: @@ -31,6 +38,11 @@ func Build(ctx context.Context, sch *schema.Schema, project Project, filesTransa } func buildModule(ctx context.Context, sch *schema.Schema, module Module, filesTransaction ModifyFilesTransaction) error { + release, err := flock.Acquire(ctx, filepath.Join(module.Dir, ".ftl-build-lock"), BuildLockTimeout) + if err != nil { + return err + } + defer release() //nolint:errcheck logger := log.FromContext(ctx).Scope(module.Module) ctx = log.ContextWithLogger(ctx, logger) @@ -40,7 +52,6 @@ func buildModule(ctx context.Context, sch *schema.Schema, module Module, filesTr } logger.Infof("Building module") - var err error switch module.Language { case "go": err = buildGoModule(ctx, sch, module, filesTransaction) diff --git a/go-runtime/ftl/ftltest/testdata/go/verbtypes/go.sum b/go-runtime/ftl/ftltest/testdata/go/verbtypes/go.sum index be1709c712..5b05f397e5 100644 --- a/go-runtime/ftl/ftltest/testdata/go/verbtypes/go.sum +++ b/go-runtime/ftl/ftltest/testdata/go/verbtypes/go.sum @@ -74,8 +74,6 @@ github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpv github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= -github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= diff --git a/go-runtime/ftl/ftltest/testdata/go/wrapped/go.sum b/go-runtime/ftl/ftltest/testdata/go/wrapped/go.sum index be1709c712..5b05f397e5 100644 --- a/go-runtime/ftl/ftltest/testdata/go/wrapped/go.sum +++ b/go-runtime/ftl/ftltest/testdata/go/wrapped/go.sum @@ -74,8 +74,6 @@ github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpv github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= -github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= diff --git a/go-runtime/ftl/testdata/go/mapper/go.sum b/go-runtime/ftl/testdata/go/mapper/go.sum index be1709c712..5b05f397e5 100644 --- a/go-runtime/ftl/testdata/go/mapper/go.sum +++ b/go-runtime/ftl/testdata/go/mapper/go.sum @@ -74,8 +74,6 @@ github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpv github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= -github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= diff --git a/go.mod b/go.mod index 6179baac41..dea3cbd948 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,6 @@ require ( github.com/docker/docker v26.1.3+incompatible github.com/docker/go-connections v0.5.0 github.com/go-logr/logr v1.4.2 - github.com/gofrs/flock v0.8.1 github.com/golang/protobuf v1.5.4 github.com/google/uuid v1.6.0 github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 @@ -134,7 +133,7 @@ require ( github.com/swaggest/refl v1.3.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect golang.org/x/crypto v0.24.0 // indirect - golang.org/x/sys v0.21.0 // indirect + golang.org/x/sys v0.21.0 golang.org/x/text v0.16.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect diff --git a/go.sum b/go.sum index 264e13c0b0..2e36fd63d0 100644 --- a/go.sum +++ b/go.sum @@ -114,8 +114,6 @@ github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpv github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= -github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= @@ -157,10 +155,6 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNU github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= @@ -215,8 +209,6 @@ github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/robertkrimen/otto v0.2.1 h1:FVP0PJ0AHIjC+N4pKCG9yCDz6LHNPCwi/GKID5pGGF0= github.com/robertkrimen/otto v0.2.1/go.mod h1:UPwtJ1Xu7JrLcZjNWN8orJaM5n5YEtqL//farB5FlRY= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/cors v1.11.0 h1:0B9GE/r9Bc2UxRMMtymBkHTenPkHDv0CW4Y98GBY+po= github.com/rs/cors v1.11.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= @@ -347,8 +339,6 @@ google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLp google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI= gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= diff --git a/internal/flock/flock.go b/internal/flock/flock.go new file mode 100644 index 0000000000..b50c05b40b --- /dev/null +++ b/internal/flock/flock.go @@ -0,0 +1,66 @@ +package flock + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "time" + + "golang.org/x/sys/unix" +) + +var ErrLocked = errors.New("locked") + +// Acquire a lock on the given path. +// +// The lock is released when the returned function is called. +func Acquire(ctx context.Context, path string, timeout time.Duration) (release func() error, err error) { + absPath, err := filepath.Abs(path) + if err != nil { + return nil, err + } + end := time.Now().Add(timeout) + for { + release, err := acquire(absPath) + if err == nil { + return release, nil + } + if !errors.Is(err, ErrLocked) { + return nil, fmt.Errorf("failed to acquire lock %s: %w", absPath, err) + } + if time.Now().After(end) { + pid, _ := os.ReadFile(absPath) + return nil, fmt.Errorf("timed out acquiring lock %s, locked by pid %s: %w", absPath, pid, err) + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(time.Second): + } + } +} + +func acquire(path string) (release func() error, err error) { + pid := os.Getpid() + fd, err := unix.Open(path, unix.O_CREAT|unix.O_RDWR|unix.O_CLOEXEC|unix.O_SYNC, 0600) + if err != nil { + return nil, fmt.Errorf("open failed: %w", err) + } + + err = unix.Flock(fd, unix.LOCK_EX|unix.LOCK_NB) + if err != nil { + _ = unix.Close(fd) + return nil, fmt.Errorf("%w: %w", ErrLocked, err) + } + + _, err = unix.Write(fd, []byte(strconv.Itoa(pid))) + if err != nil { + return nil, fmt.Errorf("write failed: %w", err) + } + return func() error { + return errors.Join(unix.Flock(fd, unix.LOCK_UN), unix.Close(fd), os.Remove(path)) + }, nil +} diff --git a/internal/flock/flock_test.go b/internal/flock/flock_test.go new file mode 100644 index 0000000000..1db2e5807b --- /dev/null +++ b/internal/flock/flock_test.go @@ -0,0 +1,28 @@ +package flock + +import ( + "context" + "path/filepath" + "testing" + + "github.com/alecthomas/assert/v2" +) + +func TestFlock(t *testing.T) { + dir := t.TempDir() + lockfile := filepath.Join(dir, "lock") + ctx := context.Background() + release, err := Acquire(ctx, lockfile, 0) + assert.NoError(t, err) + + _, err = Acquire(ctx, lockfile, 0) + assert.Error(t, err) + + err = release() + assert.NoError(t, err) + + releaseb, err := Acquire(ctx, lockfile, 0) + assert.NoError(t, err) + err = releaseb() + assert.NoError(t, err) +}