From efc799d905903e1c44489601de3643cb330b39b6 Mon Sep 17 00:00:00 2001 From: Stuart Douglas Date: Wed, 4 Sep 2024 13:09:13 +1000 Subject: [PATCH] feat: allow for build env vars (#2609) This allows for multi platform builds on go --- frontend/cli/cmd_box.go | 17 ++++++++--------- frontend/cli/cmd_build.go | 3 ++- frontend/cli/cmd_deploy.go | 9 ++++----- frontend/cli/cmd_dev.go | 13 ++++++------- go-runtime/compile/build.go | 8 ++++++-- internal/buildengine/build.go | 8 ++++---- internal/buildengine/build_go.go | 4 ++-- internal/buildengine/engine.go | 9 ++++++++- internal/exec/exec.go | 5 +++++ 9 files changed, 45 insertions(+), 31 deletions(-) diff --git a/frontend/cli/cmd_box.go b/frontend/cli/cmd_box.go index 4c1ef92e23..5ffcf594f6 100644 --- a/frontend/cli/cmd_box.go +++ b/frontend/cli/cmd_box.go @@ -87,11 +87,10 @@ func init() { } type boxCmd struct { - BaseImage string `help:"Name of the ftl-box Docker image to use as a base." default:"ftl0/ftl-box:${version}"` - Parallelism int `short:"j" help:"Number of modules to build in parallel." default:"${numcpu}"` - Compose string `help:"Path to a compose file to generate."` - Name string `arg:"" help:"Name of the project."` - Dirs []string `arg:"" help:"Base directories containing modules (defaults to modules in project config)." type:"existingdir" optional:""` + BaseImage string `help:"Name of the ftl-box Docker image to use as a base." default:"ftl0/ftl-box:${version}"` + Compose string `help:"Path to a compose file to generate."` + Name string `arg:"" help:"Name of the project."` + Build buildCmd `embed:""` } func (b *boxCmd) Help() string { @@ -117,13 +116,13 @@ Bring the box down: } func (b *boxCmd) Run(ctx context.Context, client ftlv1connect.ControllerServiceClient, projConfig projectconfig.Config) error { - if len(b.Dirs) == 0 { - b.Dirs = projConfig.AbsModuleDirs() + if len(b.Build.Dirs) == 0 { + b.Build.Dirs = projConfig.AbsModuleDirs() } - if len(b.Dirs) == 0 { + if len(b.Build.Dirs) == 0 { return errors.New("no directories specified") } - engine, err := buildengine.New(ctx, client, projConfig.Root(), b.Dirs, buildengine.Parallelism(b.Parallelism)) + engine, err := buildengine.New(ctx, client, projConfig.Root(), b.Build.Dirs, buildengine.BuildEnv(b.Build.BuildEnv), buildengine.Parallelism(b.Build.Parallelism)) if err != nil { return err } diff --git a/frontend/cli/cmd_build.go b/frontend/cli/cmd_build.go index 58ee325f7f..b2f45b9474 100644 --- a/frontend/cli/cmd_build.go +++ b/frontend/cli/cmd_build.go @@ -13,6 +13,7 @@ import ( type buildCmd struct { Parallelism int `short:"j" help:"Number of modules to build in parallel." default:"${numcpu}"` Dirs []string `arg:"" help:"Base directories containing modules (defaults to modules in project config)." type:"existingdir" optional:""` + BuildEnv []string `help:"Environment variables to set for the build."` } func (b *buildCmd) Run(ctx context.Context, client ftlv1connect.ControllerServiceClient, projConfig projectconfig.Config) error { @@ -22,7 +23,7 @@ func (b *buildCmd) Run(ctx context.Context, client ftlv1connect.ControllerServic if len(b.Dirs) == 0 { return errors.New("no directories specified") } - engine, err := buildengine.New(ctx, client, projConfig.Root(), b.Dirs, buildengine.Parallelism(b.Parallelism)) + engine, err := buildengine.New(ctx, client, projConfig.Root(), b.Dirs, buildengine.BuildEnv(b.BuildEnv), buildengine.Parallelism(b.Parallelism)) if err != nil { return err } diff --git a/frontend/cli/cmd_deploy.go b/frontend/cli/cmd_deploy.go index a2d4981b3a..27deb8932b 100644 --- a/frontend/cli/cmd_deploy.go +++ b/frontend/cli/cmd_deploy.go @@ -10,15 +10,14 @@ import ( ) type deployCmd struct { - Parallelism int `short:"j" help:"Number of modules to build in parallel." default:"${numcpu}"` - Replicas int32 `short:"n" help:"Number of replicas to deploy." default:"1"` - Dirs []string `arg:"" help:"Base directories containing modules." type:"existingdir" required:""` - NoWait bool `help:"Do not wait for deployment to complete." default:"false"` + Replicas int32 `short:"n" help:"Number of replicas to deploy." default:"1"` + NoWait bool `help:"Do not wait for deployment to complete." default:"false"` + Build buildCmd `embed:""` } func (d *deployCmd) Run(ctx context.Context, projConfig projectconfig.Config) error { client := rpc.ClientFromContext[ftlv1connect.ControllerServiceClient](ctx) - engine, err := buildengine.New(ctx, client, projConfig.Root(), d.Dirs, buildengine.Parallelism(d.Parallelism)) + engine, err := buildengine.New(ctx, client, projConfig.Root(), d.Build.Dirs, buildengine.BuildEnv(d.Build.BuildEnv), buildengine.Parallelism(d.Build.Parallelism)) if err != nil { return err } diff --git a/frontend/cli/cmd_dev.go b/frontend/cli/cmd_dev.go index c106578ab4..2eb8283a9d 100644 --- a/frontend/cli/cmd_dev.go +++ b/frontend/cli/cmd_dev.go @@ -18,21 +18,20 @@ import ( ) type devCmd struct { - Parallelism int `short:"j" help:"Number of modules to build in parallel." default:"${numcpu}"` - Dirs []string `arg:"" help:"Base directories containing modules." type:"existingdir" optional:""` Watch time.Duration `help:"Watch template directory at this frequency and regenerate on change." default:"500ms"` NoServe bool `help:"Do not start the FTL server." default:"false"` Lsp bool `help:"Run the language server." default:"false"` ServeCmd serveCmd `embed:""` InitDB bool `help:"Initialize the database and exit." default:"false"` languageServer *lsp.Server + Build buildCmd `embed:""` } func (d *devCmd) Run(ctx context.Context, projConfig projectconfig.Config) error { - if len(d.Dirs) == 0 { - d.Dirs = projConfig.AbsModuleDirs() + if len(d.Build.Dirs) == 0 { + d.Build.Dirs = projConfig.AbsModuleDirs() } - if len(d.Dirs) == 0 { + if len(d.Build.Dirs) == 0 { return errors.New("no directories specified") } @@ -78,7 +77,7 @@ func (d *devCmd) Run(ctx context.Context, projConfig projectconfig.Config) error case <-controllerReady: } - opts := []buildengine.Option{buildengine.Parallelism(d.Parallelism)} + opts := []buildengine.Option{buildengine.Parallelism(d.Build.Parallelism), buildengine.BuildEnv(d.Build.BuildEnv)} if d.Lsp { d.languageServer = lsp.NewServer(ctx) opts = append(opts, buildengine.WithListener(d.languageServer)) @@ -88,7 +87,7 @@ func (d *devCmd) Run(ctx context.Context, projConfig projectconfig.Config) error }) } - engine, err := buildengine.New(ctx, client, projConfig.Root(), d.Dirs, opts...) + engine, err := buildengine.New(ctx, client, projConfig.Root(), d.Build.Dirs, opts...) if err != nil { return err } diff --git a/go-runtime/compile/build.go b/go-runtime/compile/build.go index 17b5c86168..ee2efd465f 100644 --- a/go-runtime/compile/build.go +++ b/go-runtime/compile/build.go @@ -100,7 +100,7 @@ func buildDir(moduleDir string) string { } // Build the given module. -func Build(ctx context.Context, projectRootDir, moduleDir string, sch *schema.Schema, filesTransaction ModifyFilesTransaction) (err error) { +func Build(ctx context.Context, projectRootDir, moduleDir string, sch *schema.Schema, filesTransaction ModifyFilesTransaction, buildEnv []string) (err error) { if err := filesTransaction.Begin(); err != nil { return err } @@ -246,7 +246,11 @@ func Build(ctx context.Context, projectRootDir, moduleDir string, sch *schema.Sc } logger.Debugf("Compiling") - return exec.Command(ctx, log.Debug, mainDir, "go", "build", "-o", "../../main", ".").RunBuffered(ctx) + err = exec.CommandWithEnv(ctx, log.Debug, mainDir, buildEnv, "go", "build", "-o", "../../main", ".").RunBuffered(ctx) + if err != nil { + return fmt.Errorf("failed to compile: %w", err) + } + return nil } // CleanStubs removes all generated stubs. diff --git a/internal/buildengine/build.go b/internal/buildengine/build.go index eb64652395..48e95717ab 100644 --- a/internal/buildengine/build.go +++ b/internal/buildengine/build.go @@ -22,11 +22,11 @@ const BuildLockTimeout = time.Minute // Build a module in the given directory given the schema and module config. // // A lock file is used to ensure that only one build is running at a time. -func Build(ctx context.Context, projectRootDir string, sch *schema.Schema, module Module, filesTransaction ModifyFilesTransaction) error { - return buildModule(ctx, projectRootDir, sch, module, filesTransaction) +func Build(ctx context.Context, projectRootDir string, sch *schema.Schema, module Module, filesTransaction ModifyFilesTransaction, buildEnv []string) error { + return buildModule(ctx, projectRootDir, sch, module, filesTransaction, buildEnv) } -func buildModule(ctx context.Context, projectRootDir string, sch *schema.Schema, module Module, filesTransaction ModifyFilesTransaction) error { +func buildModule(ctx context.Context, projectRootDir string, sch *schema.Schema, module Module, filesTransaction ModifyFilesTransaction, buildEnv []string) error { release, err := flock.Acquire(ctx, filepath.Join(module.Config.Dir, ".ftl.lock"), BuildLockTimeout) if err != nil { return err @@ -46,7 +46,7 @@ func buildModule(ctx context.Context, projectRootDir string, sch *schema.Schema, switch module.Config.Language { case "go": - err = buildGoModule(ctx, projectRootDir, sch, module, filesTransaction) + err = buildGoModule(ctx, projectRootDir, sch, module, filesTransaction, buildEnv) case "java", "kotlin": err = buildJavaModule(ctx, module) case "rust": diff --git a/internal/buildengine/build_go.go b/internal/buildengine/build_go.go index 8eba01a747..ac6c5c3f0f 100644 --- a/internal/buildengine/build_go.go +++ b/internal/buildengine/build_go.go @@ -8,8 +8,8 @@ import ( "github.com/TBD54566975/ftl/go-runtime/compile" ) -func buildGoModule(ctx context.Context, projectRootDir string, sch *schema.Schema, module Module, transaction ModifyFilesTransaction) error { - if err := compile.Build(ctx, projectRootDir, module.Config.Dir, sch, transaction); err != nil { +func buildGoModule(ctx context.Context, projectRootDir string, sch *schema.Schema, module Module, transaction ModifyFilesTransaction, buildEnv []string) error { + if err := compile.Build(ctx, projectRootDir, module.Config.Dir, sch, transaction, buildEnv); err != nil { return CompilerBuildError{err: fmt.Errorf("failed to build module %q: %w", module.Config.Module, err)} } return nil diff --git a/internal/buildengine/engine.go b/internal/buildengine/engine.go index b74cf378b3..e90dadb867 100644 --- a/internal/buildengine/engine.go +++ b/internal/buildengine/engine.go @@ -75,6 +75,7 @@ type Engine struct { parallelism int listener Listener modulesToBuild *xsync.MapOf[string, bool] + buildEnv []string } type Option func(o *Engine) @@ -85,6 +86,12 @@ func Parallelism(n int) Option { } } +func BuildEnv(env []string) Option { + return func(o *Engine) { + o.buildEnv = env + } +} + // WithListener sets the event listener for the Engine. func WithListener(listener Listener) Option { return func(o *Engine) { @@ -698,7 +705,7 @@ func (e *Engine) build(ctx context.Context, moduleName string, builtModules map[ e.listener.OnBuildStarted(meta.module) } - err := Build(ctx, e.projectRoot, sch, meta.module, e.watcher.GetTransaction(meta.module.Config.Dir)) + err := Build(ctx, e.projectRoot, sch, meta.module, e.watcher.GetTransaction(meta.module.Config.Dir), e.buildEnv) if err != nil { return err } diff --git a/internal/exec/exec.go b/internal/exec/exec.go index 003e8fbdee..5e0788dfde 100644 --- a/internal/exec/exec.go +++ b/internal/exec/exec.go @@ -31,6 +31,10 @@ func Capture(ctx context.Context, dir, exe string, args ...string) ([]byte, erro } func Command(ctx context.Context, level log.Level, dir, exe string, args ...string) *Cmd { + return CommandWithEnv(ctx, level, dir, []string{}, exe, args...) +} + +func CommandWithEnv(ctx context.Context, level log.Level, dir string, env []string, exe string, args ...string) *Cmd { logger := log.FromContext(ctx) pgid, err := syscall.Getpgid(0) if err != nil { @@ -38,6 +42,7 @@ func Command(ctx context.Context, level log.Level, dir, exe string, args ...stri } logger.Tracef("exec: cd %s && %s %s", shellquote.Join(dir), exe, shellquote.Join(args...)) cmd := exec.CommandContext(ctx, exe, args...) + cmd.Env = append(cmd.Env, env...) cmd.SysProcAttr = &syscall.SysProcAttr{ Pgid: pgid, Setpgid: true,