From af445a2cd563de0e7a6ae48a7cbf39b11417d177 Mon Sep 17 00:00:00 2001 From: Alec Thomas Date: Wed, 26 Jun 2024 15:19:20 +1000 Subject: [PATCH] feat: add `ftl box` command tree (#1878) There are two commands: `ftl box build ` builds a self-contained Docker container with a set of modules. `ftl box run` runs modules inside an ftl-in-a-box container (currently does nothing). --- Dockerfile.box | 4 +- buildengine/deploy.go | 5 +- cmd/ftl/cmd_box.go | 125 ++++++++++++++++++++++++++++ cmd/ftl/main.go | 1 + common/moduleconfig/moduleconfig.go | 5 -- 5 files changed, 132 insertions(+), 8 deletions(-) create mode 100644 cmd/ftl/cmd_box.go diff --git a/Dockerfile.box b/Dockerfile.box index 1bb70ff976..fca5f065ab 100644 --- a/Dockerfile.box +++ b/Dockerfile.box @@ -41,8 +41,10 @@ COPY --from=builder /hermit/pkg/openjre-18.0.2.1_1/ ./jre/ COPY --from=builder /src/build/template template COPY --from=builder /src/build/release/ftl . RUN mkdir deployments +# Where the module artifacts are stored +RUN mkdir modules EXPOSE 8891 EXPOSE 8892 -CMD ["/root/ftl", "dev"] +CMD ["/root/ftl", "box", "run"] diff --git a/buildengine/deploy.go b/buildengine/deploy.go index c8362224e4..329a8b086b 100644 --- a/buildengine/deploy.go +++ b/buildengine/deploy.go @@ -41,7 +41,7 @@ func Deploy(ctx context.Context, module Module, replicas int32, waitForDeployOnl logger.Infof("Deploying module") moduleConfig := module.Config.Abs() - files, err := findFiles(moduleConfig) + files, err := FindFilesToDeploy(moduleConfig) if err != nil { logger.Errorf(err, "failed to find files in %s", moduleConfig) return err @@ -154,7 +154,8 @@ func loadProtoSchema(config moduleconfig.AbsModuleConfig, replicas int32) (*sche return module, nil } -func findFiles(moduleConfig moduleconfig.AbsModuleConfig) ([]string, error) { +// FindFilesToDeploy returns a list of files to deploy for the given module. +func FindFilesToDeploy(moduleConfig moduleconfig.AbsModuleConfig) ([]string, error) { var out []string for _, file := range moduleConfig.Deploy { info, err := os.Stat(file) diff --git a/cmd/ftl/cmd_box.go b/cmd/ftl/cmd_box.go new file mode 100644 index 0000000000..68affdec28 --- /dev/null +++ b/cmd/ftl/cmd_box.go @@ -0,0 +1,125 @@ +package main + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/otiai10/copy" + + "github.com/TBD54566975/ftl" + "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" + "github.com/TBD54566975/ftl/buildengine" + "github.com/TBD54566975/ftl/common/projectconfig" + "github.com/TBD54566975/ftl/internal/exec" + "github.com/TBD54566975/ftl/internal/log" +) + +const boxDockerFile = `FROM {{.BaseImage}} + +WORKDIR /root + +COPY modules /root + +EXPOSE 8891 +EXPOSE 8892 + +CMD ["/root/ftl", "dev"] + +` + +type boxCmd struct { + Build boxBuildCmd `cmd:"" help:"Build a self-contained Docker container (FTL-in-a-box) for running a set of modules."` + Run boxRunCmd `cmd:"" help:"Run an FTL-in-a-box container."` +} + +type boxRunCmd struct { +} + +func (b *boxRunCmd) Run() error { + return fmt.Errorf("not implemented") +} + +type boxBuildCmd struct { + BaseImage string `help:"Name of the ftl-box Docker image to use as a base." default:"ftl0/ftl-box:${version}"` + Parallelism int `short:"j" help:"Number of modules to build in parallel." default:"${numcpu}"` + Image string `arg:"" help:"Name of image to build."` + Dirs []string `arg:"" help:"Base directories containing modules (defaults to modules in project config)." type:"existingdir" optional:""` +} + +func (b *boxBuildCmd) Help() string { + return `` +} + +func (b *boxBuildCmd) Run(ctx context.Context, client ftlv1connect.ControllerServiceClient, projConfig projectconfig.Config) error { + if len(b.Dirs) == 0 { + b.Dirs = projConfig.AbsModuleDirs() + } + if len(b.Dirs) == 0 { + return errors.New("no directories specified") + } + engine, err := buildengine.New(ctx, client, b.Dirs, buildengine.Parallelism(b.Parallelism)) + if err != nil { + return err + } + if err := os.Setenv("GOOS", "linux"); err != nil { + return fmt.Errorf("failed to set GOOS: %w", err) + } + if err := os.Setenv("GOARCH", "amd64"); err != nil { + return fmt.Errorf("failed to set GOARCH: %w", err) + } + if err := engine.Build(ctx); err != nil { + return fmt.Errorf("build failed: %w", err) + } + + workDir, err := os.MkdirTemp("", "ftl-box-") + if err != nil { + return fmt.Errorf("failed to create temporary directory: %w", err) + } + defer os.RemoveAll(workDir) //nolint:errcheck + logger := log.FromContext(ctx) + logger.Debugf("Copying") + if err := engine.Each(func(m buildengine.Module) error { + config := m.Config.Abs() + destDir := filepath.Join(workDir, "modules", config.Module) + + // Copy deployment artefacts. + files, err := buildengine.FindFilesToDeploy(config) + if err != nil { + return err + } + files = append(files, filepath.Join(config.Dir, "ftl.toml")) + for _, file := range files { + relFile, err := filepath.Rel(config.Dir, file) + if err != nil { + return err + } + destFile := filepath.Join(destDir, relFile) + logger.Debugf(" %s -> %s", file, destFile) + if err := copy.Copy(file, destFile); err != nil { + return fmt.Errorf("failed to copy %s to %s: %w", file, destFile, err) + } + } + return nil + }); err != nil { + return err + } + baseImage := b.BaseImage + baseImageParts := strings.Split(baseImage, ":") + if len(baseImageParts) == 2 { + version := baseImageParts[1] + if !ftl.IsRelease(version) { + version = "latest" + } + baseImage = baseImageParts[0] + ":" + version + } + dockerFile := strings.ReplaceAll(boxDockerFile, "{{.BaseImage}}", baseImage) + err = os.WriteFile(filepath.Join(workDir, "Dockerfile"), []byte(dockerFile), 0600) + if err != nil { + return fmt.Errorf("failed to write Dockerfile: %w", err) + } + return exec.Command(ctx, log.Info, workDir, "docker", "build", "-t", b.Image, "--progress=plain", "--platform=linux/amd64", ".").Run() +} diff --git a/cmd/ftl/main.go b/cmd/ftl/main.go index c18fc2aead..b114725da7 100644 --- a/cmd/ftl/main.go +++ b/cmd/ftl/main.go @@ -44,6 +44,7 @@ type CLI struct { Kill killCmd `cmd:"" help:"Kill a deployment."` Schema schemaCmd `cmd:"" help:"FTL schema commands."` Build buildCmd `cmd:"" help:"Build all modules found in the specified directories."` + Box boxCmd `cmd:"" help:"Build a self-contained Docker container for running a set of module."` Deploy deployCmd `cmd:"" help:"Build and deploy all modules found in the specified directories."` Download downloadCmd `cmd:"" help:"Download a deployment."` Secret secretCmd `cmd:"" help:"Manage secrets."` diff --git a/common/moduleconfig/moduleconfig.go b/common/moduleconfig/moduleconfig.go index ba9bfd7067..dc47f0f78e 100644 --- a/common/moduleconfig/moduleconfig.go +++ b/common/moduleconfig/moduleconfig.go @@ -105,11 +105,6 @@ func (c ModuleConfig) Abs() AbsModuleConfig { return AbsModuleConfig(clone) } -// AbsDeployDir returns the absolute path to the deploy directory. -func (c ModuleConfig) AbsDeployDir() string { - return filepath.Join(c.Dir, c.DeployDir) -} - func setConfigDefaults(moduleDir string, config *ModuleConfig) error { if config.Realm == "" { config.Realm = "home"