From 8b424bd4dd676f07cbdb11ece40b89353c7cb194 Mon Sep 17 00:00:00 2001 From: Matt Toohey Date: Tue, 14 May 2024 08:37:11 +1000 Subject: [PATCH] test: make TestHashRing() more reliable (#1467) This might make the test more reliable. fixes https://github.com/TBD54566975/ftl/issues/1446 We may have been hitting a deadlock situation where cronjobs were setting up their observation of channels and calling `clock.Now()` while the test tried to increment the mock clock (exact case is detailed more in https://github.com/TBD54566975/ftl/issues/1368). --- backend/controller/cronjobs/cronjobs_test.go | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/backend/controller/cronjobs/cronjobs_test.go b/backend/controller/cronjobs/cronjobs_test.go index dff802f035..7643fc86b1 100644 --- a/backend/controller/cronjobs/cronjobs_test.go +++ b/backend/controller/cronjobs/cronjobs_test.go @@ -52,10 +52,6 @@ func TestHashRing(t *testing.T) { lock: sync.Mutex{}, attemptCountMap: map[string]int{}, } - - verbCallCount := map[string]int{} - verbCallCountLock := sync.Mutex{} - moduleName := "initial" jobsToCreate := newJobs(t, moduleName, "*/10 * * * * * *", mockDal.clock, 100) @@ -68,15 +64,13 @@ func TestHashRing(t *testing.T) { assert.NoError(t, err) controllers := newControllers(ctx, 20, mockDal, func() clock.Clock { return clock.NewMock() }, func(ctx context.Context, r *connect.Request[ftlv1.CallRequest], o optional.Option[model.RequestKey], s string) (*connect.Response[ftlv1.CallResponse], error) { - verbRef := schema.RefFromProto(r.Msg.Verb) - - verbCallCountLock.Lock() - verbCallCount[verbRef.Name]++ - verbCallCountLock.Unlock() - return &connect.Response[ftlv1.CallResponse]{}, nil }) + // This should give time for each controller to start watching its own mock clock + // If we don;t wait here, we might hit a race condition outlined in issue #1368 + time.Sleep(time.Millisecond * 100) + // progress time for each controller one at a time, noting which verbs got attempted each time // to build a map of verb to controller keys controllersForVerbs := map[string][]model.ControllerKey{}