Skip to content

Commit

Permalink
feat: only rebuild dependent modules on schema changes (#819)
Browse files Browse the repository at this point in the history
Fixes #792 

Example of rebuilding `shipping` and `checkout` because they depend on
`cart` schema

```bash
warn: Detected change in file /Users/wesbillman/dev/ftl/examples/online-boutique/services/cart/cart.go
info: Creating deployment for module cart
info: Building Go module 'cart'
info: Generating external modules
info: Tidying go.mod
info: Extracting schema
info: Generating main module
info: Compiling
info: Uploading 0/1 files
info: Created deployment cart-548ccb3647
warn: Rebuilding "shipping" because of "cart" schema changes
warn: Rebuilding "checkout" because of "cart" schema changes
info: Creating deployment for module checkout
info: Building Go module 'checkout'
info: Generating external modules
info: Tidying go.mod
info: Extracting schema
info: Generating main module
info: Compiling
info: Uploading 0/1 files
info: Created deployment checkout-09fe50cfa3
info: Creating deployment for module shipping
info: Building Go module 'shipping'
info: Generating external modules
info: Tidying go.mod
info: Extracting schema
info: Generating main module
info: Compiling
info: Uploading 0/1 files
info: Created deployment shipping-1d5aeff6d0
```
  • Loading branch information
wesbillman authored Jan 19, 2024
1 parent 107e717 commit 4e24fe1
Showing 1 changed file with 45 additions and 12 deletions.
57 changes: 45 additions & 12 deletions cmd/ftl/cmd_dev.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,15 @@ import (

"github.com/TBD54566975/ftl/backend/common/log"
"github.com/TBD54566975/ftl/backend/common/moduleconfig"
"github.com/TBD54566975/ftl/backend/schema"
ftlv1 "github.com/TBD54566975/ftl/protos/xyz/block/ftl/v1"
"github.com/TBD54566975/ftl/protos/xyz/block/ftl/v1/ftlv1connect"
)

type moduleFolderInfo struct {
moduleName string
fileHashes map[string][]byte
schema *schema.Module
}

type devCmd struct {
Expand Down Expand Up @@ -57,10 +59,38 @@ func (m *moduleMap) SetModule(dir string, module *moduleFolderInfo) {
(*m)[dir] = module
}

func (m *moduleMap) ForceRebuildFromDependent(module string) {
func (m *moduleMap) RebuildDependentModules(ctx context.Context, sch *schema.Module) {
logger := log.FromContext(ctx)
var changedModuleDir string
for dir, moduleInfo := range *m {
if moduleInfo.moduleName != module {
(*m).ForceRebuild(dir)
if moduleInfo.moduleName == sch.Name {
changedModuleDir = dir
}
}

// no module found, nothing to do
if (*m)[changedModuleDir] == nil {
return
}

oldSchema := (*m)[changedModuleDir].schema
(*m)[changedModuleDir].schema = sch

// no change in schema, nothing to do
if oldSchema == nil || oldSchema.String() == sch.String() {
return
}

for dir, moduleInfo := range *m {
if moduleInfo.schema == nil {
continue
}

for _, imp := range moduleInfo.schema.Imports() {
if imp == sch.Name {
logger.Warnf("Rebuilding %q due to %q schema changes", moduleInfo.moduleName, (*m)[changedModuleDir].moduleName)
(*m).ForceRebuild(dir)
}
}
}
}
Expand All @@ -69,7 +99,7 @@ func (d *devCmd) Run(ctx context.Context, client ftlv1connect.ControllerServiceC
logger := log.FromContext(ctx)
logger.Infof("Watching %s for FTL modules", d.BaseDir)

schemaChanges := make(chan string, 64)
schemaChanges := make(chan *schema.Module, 64)
modules := make(moduleMap)

wg, ctx := errgroup.WithContext(ctx)
Expand Down Expand Up @@ -118,14 +148,14 @@ func (d *devCmd) Run(ctx context.Context, client ftlv1connect.ControllerServiceC
}

select {
case moduleName := <-schemaChanges:
modules.ForceRebuildFromDependent(moduleName)
case module := <-schemaChanges:
modules.RebuildDependentModules(ctx, module)

drainLoop: // Drain all messages from the channel to avoid extra redeploys
for {
select {
case moduleName := <-schemaChanges:
modules.ForceRebuildFromDependent(moduleName)
case module := <-schemaChanges:
modules.RebuildDependentModules(ctx, module)
default:
break drainLoop
}
Expand All @@ -137,7 +167,7 @@ func (d *devCmd) Run(ctx context.Context, client ftlv1connect.ControllerServiceC
}
}

func (d *devCmd) watchForSchemaChanges(ctx context.Context, client ftlv1connect.ControllerServiceClient, schemaChanges chan string) error {
func (d *devCmd) watchForSchemaChanges(ctx context.Context, client ftlv1connect.ControllerServiceClient, schemaChanges chan *schema.Module) error {
logger := log.FromContext(ctx)
for {
stream, err := client.PullSchema(ctx, connect.NewRequest(&ftlv1.PullSchemaRequest{}))
Expand All @@ -147,9 +177,12 @@ func (d *devCmd) watchForSchemaChanges(ctx context.Context, client ftlv1connect.

for stream.Receive() {
msg := stream.Msg()
if msg.ChangeType == ftlv1.DeploymentChangeType_DEPLOYMENT_CHANGED {
logger.Warnf("Schema change detected for module %s", msg.ModuleName)
schemaChanges <- msg.ModuleName
if msg.ChangeType == ftlv1.DeploymentChangeType_DEPLOYMENT_ADDED || msg.ChangeType == ftlv1.DeploymentChangeType_DEPLOYMENT_CHANGED {
module, err := schema.ModuleFromProto(msg.Schema)
if err != nil {
return err
}
schemaChanges <- module
}
}

Expand Down

0 comments on commit 4e24fe1

Please sign in to comment.