Skip to content

Commit

Permalink
fix: restore deployed log (#2722)
Browse files Browse the repository at this point in the history
  • Loading branch information
stuartwdouglas authored Sep 19, 2024
1 parent 92ea9e9 commit a89bf83
Showing 1 changed file with 7 additions and 7 deletions.
14 changes: 7 additions & 7 deletions backend/controller/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -1729,7 +1729,6 @@ func (s *Service) getDeploymentLogger(ctx context.Context, deploymentKey model.D

// Periodically sync the routing table from the DB.
func (s *Service) syncRoutes(ctx context.Context) (ret time.Duration, err error) {
logger := log.FromContext(ctx)
deployments, err := s.dal.GetActiveDeployments(ctx)
if errors.Is(err, libdal.ErrNotFound) {
deployments = []dal.Deployment{}
Expand All @@ -1745,15 +1744,16 @@ func (s *Service) syncRoutes(ctx context.Context) (ret time.Duration, err error)
old := s.routes.Load()
newRoutes := map[string]Route{}
for _, v := range deployments {
logger.Tracef("processing deployment %s for route table", v.Key.String())
deploymentLogger := s.getDeploymentLogger(ctx, v.Key)
deploymentLogger.Tracef("processing deployment %s for route table", v.Key.String())
// Deployments are in order, oldest to newest
// If we see a newer one overwrite an old one that means the new one is read
// And we set its replicas to zero
// It may seem a bit odd to do this here but this is where we are actually updating the routing table
// Which is what makes as a deployment 'live' from a clients POV
optURI, err := s.runnerScaling.GetEndpointForDeployment(ctx, v.Module, v.Key.String())
if err != nil {
logger.Errorf(err, "Failed to get updated endpoint for deployment %s", v.Key.String())
deploymentLogger.Errorf(err, "Failed to get updated endpoint for deployment %s", v.Key.String())
continue
} else if uri, ok := optURI.Get(); ok {
// Check if this is a new route
Expand All @@ -1763,20 +1763,20 @@ func (s *Service) syncRoutes(ctx context.Context) (ret time.Duration, err error)
// Kube deployments can take a while to come up, so we don't want to add them to the routing table until they are ready.
_, err := s.clientsForEndpoint(targetEndpoint).verb.Ping(ctx, connect.NewRequest(&ftlv1.PingRequest{}))
if err != nil {
logger.Tracef("Unable to ping %s, not adding to route table", v.Key.String())
deploymentLogger.Tracef("Unable to ping %s, not adding to route table", v.Key.String())
continue
}
logger.Debugf("Adding %s to route table", v.Key.String())
deploymentLogger.Infof("Deployed %s", v.Key.String())
}
if prev, ok := newRoutes[v.Module]; ok {
// We have already seen a route for this module, the existing route must be an old one
// as the deployments are in order
// We have a new route ready to go, so we can just set the old one to 0 replicas
// Do this in a TX so it doesn't happen until the route table is updated
logger.Debugf("Setting %s to zero replicas", v.Key.String())
deploymentLogger.Debugf("Setting %s to zero replicas", v.Key.String())
err := tx.SetDeploymentReplicas(ctx, prev.Deployment, 0)
if err != nil {
logger.Errorf(err, "Failed to set replicas to 0 for deployment %s", prev.Deployment.String())
deploymentLogger.Errorf(err, "Failed to set replicas to 0 for deployment %s", prev.Deployment.String())
}
}
newRoutes[v.Module] = Route{Module: v.Module, Deployment: v.Key, Endpoint: targetEndpoint}
Expand Down

0 comments on commit a89bf83

Please sign in to comment.