Add retry for reconcile error (#232)

* retry * ci * Update pkg/utils/event_source.go Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Apply suggestions from code review Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * ci * fix ci * Update api/v1alpha1/zz_generated.deepcopy.go Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix ci * fix ci * fix ci --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
streamnative · Aug 21, 2024 · 35cec2f · 35cec2f
1 parent 782fb18
commit 35cec2f
Show file tree

Hide file tree

Showing 18 changed files with 1,160 additions and 38 deletions.
diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml
@@ -44,10 +44,10 @@ jobs:
           docker rmi $(docker images -q) -f
           df -h
 
-      - name: Set up Go 1.21
+      - name: Set up Go 1.22.6
         uses: actions/setup-go@v3
         with:
-          go-version: '1.21'
+          go-version: '1.22.6'
         id: go
 
       - name: Set up Git token

diff --git a/.golangci.yml b/.golangci.yml
@@ -23,7 +23,6 @@ linters:
   - errcheck
   - exportloopref
   # - gci
-  - goconst
   - gocritic
   - gocyclo
   # - godot
@@ -33,7 +32,6 @@ linters:
   - gosec
   - gosimple
   - govet
-  - ifshort
   - importas
   - ineffassign
   - misspell
@@ -42,7 +40,6 @@ linters:
   - nolintlint
   - prealloc
   - predeclared
-  - revive
   - rowserrcheck
   - staticcheck
   - structcheck
@@ -52,7 +49,6 @@ linters:
   - unconvert
   - unparam
   - varcheck
-  - whitespace
 
 linters-settings:
   godot:
@@ -63,12 +59,9 @@ linters-settings:
     exclude:
     - '^ \+.*'
     - '^ ANCHOR.*'
-  ifshort:
-    # Maximum length of variable declaration measured in number of characters, after which linter won't suggest using short syntax.
-    max-decl-chars: 50
   gci:
     local-prefixes: "github.com/streamnative"
-    
+
   importas:
     no-unaliased: true
     alias:

diff --git a/.licenserc.yaml b/.licenserc.yaml
@@ -29,6 +29,8 @@ header:
     - '.github/openshift/'
     - 'PROJECT'
     - '**/go.mod'
+    - '**/go.work'
+    - '**/go.work.sum'
     - '**/go.sum'
     - '.husky/**'
     - 'hack/**'

diff --git a/controllers/pulsarconnection_controller.go b/controllers/pulsarconnection_controller.go
@@ -48,6 +48,7 @@ type PulsarConnectionReconciler struct {
 	Log                logr.Logger
 	Recorder           record.EventRecorder
 	PulsarAdminCreator admin.PulsarAdminCreator
+	Retryer            *utils.ReconcileRetryer
 }
 
 //nolint:lll
@@ -109,11 +110,12 @@ func (r *PulsarConnectionReconciler) Reconcile(ctx context.Context, req ctrl.Req
 
 	r.Log.Info("Reconciling PulsarConnection", "name", pulsarConnection.Name, "namespace", pulsarConnection.Namespace)
 
-	reconciler := connection.MakeReconciler(r.Log, r.Client, r.PulsarAdminCreator, pulsarConnection)
+	reconciler := connection.MakeReconciler(r.Log, r.Client, r.PulsarAdminCreator, pulsarConnection, r.Retryer)
 	if err := reconciler.Observe(ctx); err != nil {
 		return ctrl.Result{}, err
 	}
 	if err := reconciler.Reconcile(ctx); err != nil {
+		r.Retryer.CreateIfAbsent(pulsarConnection)
 		return ctrl.Result{}, err
 	}
 
@@ -241,8 +243,10 @@ func (r *PulsarConnectionReconciler) SetupWithManager(mgr ctrl.Manager, options
 		Watches(&source.Kind{Type: &corev1.Secret{}},
 			handler.EnqueueRequestsFromMapFunc(r.findSecretsForConnection),
 			builder.WithPredicates(secretPredicate())).
+		Watches(&source.Channel{Source: r.Retryer.Source()}, &handler.EnqueueRequestForObject{}).
 		WithOptions(options).
 		Complete(r)
+
 }
 
 func (r *PulsarConnectionReconciler) findSecretsForConnection(secret client.Object) []reconcile.Request {

diff --git a/go.mod b/go.mod
@@ -6,6 +6,7 @@ require (
 	github.com/apache/pulsar-client-go v0.12.1
 	github.com/go-logr/logr v1.2.1
 	github.com/onsi/ginkgo v1.16.5
+	github.com/onsi/ginkgo/v2 v2.1.3
 	github.com/onsi/gomega v1.19.0
 	github.com/xhit/go-str2duration/v2 v2.1.0
 	go.uber.org/zap v1.19.1

diff --git a/go.sum b/go.sum
@@ -258,6 +258,7 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -381,13 +382,15 @@ github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
+github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
 github.com/onsi/ginkgo/v2 v2.1.3 h1:e/3Cwtogj0HA+25nMP1jCMDIf8RtRYbGwGGuBIFztkc=
 github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
 github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw=
 github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
@@ -624,6 +627,7 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=

diff --git a/go.work b/go.work
@@ -0,0 +1,6 @@
+go 1.22.6
+
+use (
+	.
+	tests
+)