From c41408bc246056e9afe051be0969665840385998 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 23 Nov 2023 08:15:24 -0800 Subject: [PATCH] feat: add ftl dev command to watch and deploy (#630) --- backend/common/moduleconfig/config.go | 30 +++ cmd/ftl/cmd_build.go | 10 +- cmd/ftl/cmd_deploy.go | 24 --- cmd/ftl/cmd_dev.go | 259 ++++++++++++++++++++++++++ cmd/ftl/main.go | 1 + go.mod | 1 + go.sum | 2 + 7 files changed, 295 insertions(+), 32 deletions(-) create mode 100644 cmd/ftl/cmd_dev.go diff --git a/backend/common/moduleconfig/config.go b/backend/common/moduleconfig/config.go index bfcbeb93b7..ad0dfae2b4 100644 --- a/backend/common/moduleconfig/config.go +++ b/backend/common/moduleconfig/config.go @@ -17,6 +17,7 @@ type ModuleConfig struct { Deploy []string `toml:"deploy"` DeployDir string `toml:"deploy-dir"` Schema string `toml:"schema"` + Watch []string `toml:"watch"` } // LoadConfig from a directory. @@ -27,5 +28,34 @@ func LoadConfig(dir string) (ModuleConfig, error) { if err != nil { return ModuleConfig{}, errors.WithStack(err) } + setConfigDefaults(&config) return config, nil } + +func setConfigDefaults(config *ModuleConfig) { + switch config.Language { + case "kotlin": + if config.Build == "" { + config.Build = "mvn compile" + } + if config.DeployDir == "" { + config.DeployDir = "target" + } + if len(config.Deploy) == 0 { + config.Deploy = []string{"main", "classes", "dependency", "classpath.txt"} + } + if config.Schema == "" { + config.Schema = "schema.pb" + } + if len(config.Watch) == 0 { + config.Watch = []string{"pom.xml", "src/**", "target/generated-sources"} + } + case "go": + if config.DeployDir == "" { + config.DeployDir = "build" + } + if len(config.Deploy) == 0 { + config.Deploy = []string{"main", "schema.pb"} + } + } +} diff --git a/cmd/ftl/cmd_build.go b/cmd/ftl/cmd_build.go index e70b917931..6c287d6a65 100644 --- a/cmd/ftl/cmd_build.go +++ b/cmd/ftl/cmd_build.go @@ -32,16 +32,10 @@ func (b *buildCmd) Run(ctx context.Context) error { func (b *buildCmd) buildKotlin(ctx context.Context, config moduleconfig.ModuleConfig) error { logger := log.FromContext(ctx) - buildCmd := config.Build - - if buildCmd == "" { - buildCmd = "mvn compile" - } - logger.Infof("Building kotlin module '%s'", config.Module) - logger.Infof("Using build command '%s'", buildCmd) + logger.Infof("Using build command '%s'", config.Build) - err := exec.Command(ctx, logger.GetLevel(), b.ModuleDir, "bash", "-c", buildCmd).Run() + err := exec.Command(ctx, logger.GetLevel(), b.ModuleDir, "bash", "-c", config.Build).Run() if err != nil { return errors.WithStack(err) } diff --git a/cmd/ftl/cmd_deploy.go b/cmd/ftl/cmd_deploy.go index 155ef44bc3..7fdd1b6677 100644 --- a/cmd/ftl/cmd_deploy.go +++ b/cmd/ftl/cmd_deploy.go @@ -36,8 +36,6 @@ func (d *deployCmd) Run(ctx context.Context, client ftlv1connect.ControllerServi } logger.Infof("Creating deployment for module %s", config.Module) - setConfigDefaults(&config) - if len(config.Deploy) == 0 { return errors.Errorf("no deploy paths defined in config") } @@ -219,25 +217,3 @@ func findFilesInDir(dir string) ([]string, error) { return nil })) } - -func setConfigDefaults(config *moduleconfig.ModuleConfig) { - switch config.Language { - case "kotlin": - if config.DeployDir == "" { - config.DeployDir = "target" - } - if len(config.Deploy) == 0 { - config.Deploy = []string{"main", "classes", "dependency", "classpath.txt"} - } - if config.Schema == "" { - config.Schema = "schema.pb" - } - case "go": - if config.DeployDir == "" { - config.DeployDir = "build" - } - if len(config.Deploy) == 0 { - config.Deploy = []string{"main", "schema.pb"} - } - } -} diff --git a/cmd/ftl/cmd_dev.go b/cmd/ftl/cmd_dev.go new file mode 100644 index 0000000000..7edff3fe4d --- /dev/null +++ b/cmd/ftl/cmd_dev.go @@ -0,0 +1,259 @@ +package main + +import ( + "bufio" + "context" + "io/fs" + "os" + "path" + "path/filepath" + "strings" + "time" + + "github.com/alecthomas/errors" + "github.com/bmatcuk/doublestar/v4" + + "github.com/TBD54566975/ftl/backend/common/log" + "github.com/TBD54566975/ftl/backend/common/moduleconfig" + "github.com/TBD54566975/ftl/protos/xyz/block/ftl/v1/ftlv1connect" +) + +type moduleFolderInfo struct { + NumFiles int + LastModTime time.Time +} + +type devCmd struct { + BaseDir string `arg:"" help:"Directory to watch for FTL modules" type:"existingdir" default:"."` + Watch time.Duration `help:"Watch template directory at this frequency and regenerate on change." default:"500ms"` + modules map[string]moduleFolderInfo + client ftlv1connect.ControllerServiceClient +} + +func (d *devCmd) Run(ctx context.Context, client ftlv1connect.ControllerServiceClient) error { + logger := log.FromContext(ctx) + logger.Infof("Watching %s for FTL modules", d.BaseDir) + + d.modules = make(map[string]moduleFolderInfo) + d.client = client + + lastScanTime := time.Now() + for { + iterationStartTime := time.Now() + + tomls, err := d.getTomls() + if err != nil { + return errors.WithStack(err) + } + + d.addOrRemoveModules(tomls) + + for dir := range d.modules { + currentModule := d.modules[dir] + err := d.updateFileInfo(dir) + if err != nil { + return errors.WithStack(err) + } + + if currentModule.NumFiles != d.modules[dir].NumFiles || d.modules[dir].LastModTime.After(lastScanTime) { + deploy := deployCmd{ + Replicas: 1, + ModuleDir: dir, + } + err = deploy.Run(ctx, client) + if err != nil { + logger.Errorf(err, "Error deploying module %s. Will retry", dir) + delete(d.modules, dir) + } + } + } + + lastScanTime = iterationStartTime + select { + case <-time.After(d.Watch): + case <-ctx.Done(): + return nil + } + } +} + +func (d *devCmd) getTomls() ([]string, error) { + baseDir := d.BaseDir + ignores := loadGitIgnore(os.DirFS(baseDir), ".") + tomls := []string{} + + err := walkDir(baseDir, ignores, func(srcPath string, d fs.DirEntry) error { + if filepath.Base(srcPath) == "ftl.toml" { + tomls = append(tomls, srcPath) + return errSkip // Return errSkip to stop recursion in this branch + } + return nil + }) + if err != nil { + return nil, errors.WithStack(err) + } + + return tomls, nil +} + +func (d *devCmd) addOrRemoveModules(tomls []string) { + for _, toml := range tomls { + dir := filepath.Dir(toml) + if _, ok := d.modules[dir]; !ok { + d.modules[dir] = moduleFolderInfo{ + LastModTime: time.Now(), + } + } + } + + for dir := range d.modules { + found := false + for _, toml := range tomls { + if filepath.Dir(toml) == dir { + found = true + break + } + } + if !found { + delete(d.modules, dir) // Remove deleted module from d.modules + } + } +} + +func (d *devCmd) updateFileInfo(dir string) error { + config, err := moduleconfig.LoadConfig(dir) + if err != nil { + return errors.WithStack(err) + } + + ignores := loadGitIgnore(os.DirFS(dir), ".") + d.modules[dir] = moduleFolderInfo{} + + err = walkDir(dir, ignores, func(srcPath string, entry fs.DirEntry) error { + for _, pattern := range config.Watch { + relativePath, err := filepath.Rel(dir, srcPath) + if err != nil { + return errors.WithStack(err) + } + + match, err := doublestar.PathMatch(pattern, relativePath) + if err != nil { + return errors.WithStack(err) + } + + if match && !entry.IsDir() { + fileInfo, err := entry.Info() + if err != nil { + return errors.WithStack(err) + } + + module := d.modules[dir] + module.NumFiles++ + if fileInfo.ModTime().After(module.LastModTime) { + module.LastModTime = fileInfo.ModTime() + } + d.modules[dir] = module + } + } + + return nil + }) + + return err +} + +// errSkip is returned by walkDir to skip a file or directory. +var errSkip = errors.New("skip directory") + +// Depth-first walk of dir executing fn after each entry. +func walkDir(dir string, ignores []string, fn func(path string, d fs.DirEntry) error) error { + dirInfo, err := os.Stat(dir) + if err != nil { + return errors.WithStack(err) + } + if err = fn(dir, fs.FileInfoToDirEntry(dirInfo)); err != nil { + if errors.Is(err, errSkip) { + return nil + } + return errors.WithStack(err) + } + entries, err := os.ReadDir(dir) + if err != nil { + return errors.WithStack(err) + } + + var dirs []os.DirEntry + + // Process files first, then recurse into directories. + for _, entry := range entries { + fullPath := filepath.Join(dir, entry.Name()) + + // Check if the path matches any ignore pattern + shouldIgnore := false + for _, pattern := range ignores { + match, err := doublestar.PathMatch(pattern, fullPath) + if err != nil { + return errors.WithStack(err) + } + if match { + shouldIgnore = true + break + } + } + + if shouldIgnore { + continue // Skip this entry + } + + if entry.IsDir() { + dirs = append(dirs, entry) + } else { + if err = fn(fullPath, entry); err != nil { + if errors.Is(err, errSkip) { + // If errSkip is found in a file, skip the remaining files in this directory + return nil + } + return errors.WithStack(err) + } + } + } + + // Then, recurse into subdirectories + for _, dirEntry := range dirs { + dirPath := filepath.Join(dir, dirEntry.Name()) + ignores = append(ignores, loadGitIgnore(os.DirFS(dirPath), ".")...) + if err := walkDir(dirPath, ignores, fn); err != nil { + if errors.Is(err, errSkip) { + return errSkip // Propagate errSkip upwards to stop this branch of recursion + } + return errors.WithStack(err) + } + } + return nil +} + +func loadGitIgnore(root fs.FS, dir string) []string { + ignore := []string{ + "**/.*", + "**/.*/**", + } + r, err := root.Open(path.Join(dir, ".gitignore")) + if err != nil { + return nil + } + lr := bufio.NewScanner(r) + for lr.Scan() { + line := lr.Text() + line = strings.TrimSpace(line) + if line == "" || line[0] == '#' || line[0] == '!' { // We don't support negation. + continue + } + if strings.HasSuffix(line, "/") { + line = path.Join("**", line, "**/*") + } else if !strings.ContainsRune(line, '/') { + line = path.Join("**", line) + } + ignore = append(ignore, line) + } + return ignore +} diff --git a/cmd/ftl/main.go b/cmd/ftl/main.go index 7d79b9458b..9542991daa 100644 --- a/cmd/ftl/main.go +++ b/cmd/ftl/main.go @@ -30,6 +30,7 @@ type CLI struct { Status statusCmd `cmd:"" help:"Show FTL status."` Init initCmd `cmd:"" help:"Initialize a new FTL module."` + Dev devCmd `cmd:"" help:"Watch a directory for FTL modules and hot reload them."` PS psCmd `cmd:"" help:"List deployments."` Serve serveCmd `cmd:"" help:"Start the FTL server."` Call callCmd `cmd:"" help:"Call an FTL function."` diff --git a/go.mod b/go.mod index 7b566b4f7e..d5ad15ac26 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/alecthomas/kong v0.8.1 github.com/alecthomas/kong-toml v0.0.1 github.com/amacneil/dbmate/v2 v2.7.0 + github.com/bmatcuk/doublestar/v4 v4.6.1 github.com/go-logr/logr v1.2.4 github.com/golang/protobuf v1.5.3 github.com/google/uuid v1.4.0 diff --git a/go.sum b/go.sum index 8414bf387e..e83dcc6a50 100644 --- a/go.sum +++ b/go.sum @@ -32,6 +32,8 @@ github.com/amacneil/dbmate/v2 v2.7.0 h1:aaTyfVPxf01KyIx5E0i/OATygrit2hjivyKSX0YS github.com/amacneil/dbmate/v2 v2.7.0/go.mod h1:I13evRylGros6OXuJij+oskvr+YMaPvSD9Z7PNucXWA= github.com/beevik/etree v1.2.0 h1:l7WETslUG/T+xOPs47dtd6jov2Ii/8/OjCldk5fYfQw= github.com/beevik/etree v1.2.0/go.mod h1:aiPf89g/1k3AShMVAzriilpcE4R/Vuor90y83zVZWFc= +github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I= +github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/bool64/dev v0.2.31 h1:OS57EqYaYe2M/2bw9uhDCIFiZZwywKFS/4qMLN6JUmQ= github.com/bool64/dev v0.2.31/go.mod h1:iJbh1y/HkunEPhgebWRNcs8wfGq7sjvJ6W5iabL8ACg= github.com/bool64/shared v0.1.5 h1:fp3eUhBsrSjNCQPcSdQqZxxh9bBwrYiZ+zOKFkM0/2E=