feat: support ns offload policy (#250)

* chore: upgrade pulsar-client-go pkg

Signed-off-by: ericsyh <[email protected]>

* feat: support offloadThreshold policy on Namespace

Signed-off-by: ericsyh <[email protected]>

* add doc

Signed-off-by: ericsyh <[email protected]>

* update golint

Signed-off-by: ericsyh <[email protected]>

* fix lint

Signed-off-by: ericsyh <[email protected]>

* fix golint

Signed-off-by: ericsyh <[email protected]>

---------

Signed-off-by: ericsyh <[email protected]>

.golangci.yml

@@ -17,7 +17,8 @@ linters:
   enable:
   - asciicheck
   - bodyclose
-  - deadcode
+  - unused
+  # - deadcode
   # - depguard
   - dogsled
   - errcheck
@@ -42,13 +43,13 @@ linters:
   - predeclared
   - rowserrcheck
   - staticcheck
-  - structcheck
+  # - structcheck
   - stylecheck
   - thelper
   - typecheck
   - unconvert
   - unparam
-  - varcheck
+  # - varcheck
 
 linters-settings:
   godot:

api/v1alpha1/pulsarnamespace_types.go

@@ -99,6 +99,16 @@ type PulsarNamespaceSpec struct {
 	// +optional
 	BacklogQuotaType *string `json:"backlogQuotaType,omitempty"`
 
+	// OffloadThresholdTime specifies the time limit for message offloading.
+	// Messages older than this limit will be offloaded to the tiered storage.
+	// +optional
+	OffloadThresholdTime *utils.Duration `json:"offloadThresholdTime,omitempty"`
+
+	// OffloadThresholdSize specifies the size limit for message offloading.
+	// When the limit is reached, older messages will be offloaded to the tiered storage.
+	// +optional
+	OffloadThresholdSize *resource.Quantity `json:"offloadThresholdSize,omitempty"`
+
 	// GeoReplicationRefs is a list of references to PulsarGeoReplication resources,
 	// used to configure geo-replication for this namespace.
 	// This is **ONLY** used when you are using PulsarGeoReplication for setting up geo-replication

docs/pulsar_namespace.md

@@ -22,6 +22,8 @@ The `PulsarNamespace` resource defines a namespace in a Pulsar cluster. It allow
 | `backlogQuotaLimitSize` | Size limit for message backlog. When the limit is reached, older messages will be removed or handled according to the retention policy. | No |
 | `backlogQuotaRetentionPolicy` | Retention policy for messages when backlog quota is exceeded. Options: "producer_request_hold", "producer_exception", or "consumer_backlog_eviction". | No |
 | `backlogQuotaType` | Controls how the backlog quota is enforced. Options: "destination_storage" (limits backlog by size in bytes), "message_age" (limits by time). | No |
+| `offloadThresholdTime` | Time limit for message offloading. Messages older than this limit will be offloaded to the tiered storage. | No |
+| `offloadThresholdSize` | Size limit for message offloading. When the limit is reached, older messages will be offloaded to the tiered storage. | No |
 | `geoReplicationRefs` | List of references to PulsarGeoReplication resources, used to configure geo-replication for this namespace. Use only when using PulsarGeoReplication for setting up geo-replication between two Pulsar instances. | No |
 | `replicationClusters` | List of clusters to which the namespace is replicated. Use only if replicating clusters within the same Pulsar instance. | No |
 

go.mod

@@ -3,56 +3,47 @@ module github.com/streamnative/pulsar-resources-operator
 go 1.22.6
 
 require (
-	github.com/apache/pulsar-client-go v0.12.1
-	github.com/go-logr/logr v1.2.3
+	github.com/apache/pulsar-client-go v0.13.0-candidate-1.0.20240827063209-953d9eab0794
+	github.com/go-logr/logr v1.4.1
 	github.com/onsi/ginkgo v1.16.5
-	github.com/onsi/ginkgo/v2 v2.1.6
-	github.com/onsi/gomega v1.20.1
+	github.com/onsi/ginkgo/v2 v2.6.0
+	github.com/onsi/gomega v1.24.1
 	github.com/xhit/go-str2duration/v2 v2.1.0
-	go.uber.org/zap v1.21.0
-	k8s.io/api v0.25.15
-	k8s.io/apiextensions-apiserver v0.25.15
-	k8s.io/apimachinery v0.25.15
-	k8s.io/client-go v0.25.15
-	k8s.io/component-base v0.25.15
-	k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed
-	sigs.k8s.io/controller-runtime v0.13.2
+	go.uber.org/zap v1.24.0
+	k8s.io/api v0.26.0
+	k8s.io/apiextensions-apiserver v0.26.0
+	k8s.io/apimachinery v0.26.0
+	k8s.io/client-go v0.26.0
+	k8s.io/component-base v0.26.0
+	k8s.io/utils v0.0.0-20221128185143-99ec85e7a448
+	sigs.k8s.io/controller-runtime v0.14.0
 )
 
 require (
-	cloud.google.com/go v0.97.0 // indirect
 	github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
 	github.com/99designs/keyring v1.2.1 // indirect
-	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
-	github.com/Azure/go-autorest/autorest v0.11.27 // indirect
-	github.com/Azure/go-autorest/autorest/adal v0.9.20 // indirect
-	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
-	github.com/Azure/go-autorest/logger v0.2.1 // indirect
-	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
-	github.com/PuerkitoBio/purell v1.1.1 // indirect
-	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/danieljoos/wincred v1.1.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dvsekhvalnov/jose2go v1.6.0 // indirect
-	github.com/emicklei/go-restful/v3 v3.8.0 // indirect
+	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
-	github.com/fsnotify/fsnotify v1.5.4 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/go-logr/zapr v1.2.3 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
-	github.com/go-openapi/jsonreference v0.19.5 // indirect
+	github.com/go-openapi/jsonreference v0.20.0 // indirect
 	github.com/go-openapi/swag v0.19.14 // indirect
 	github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
-	github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
+	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/gnostic v0.5.7-v3refs // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
-	github.com/google/uuid v1.3.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
@@ -72,22 +63,21 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
-	golang.org/x/crypto v0.24.0 // indirect
 	golang.org/x/net v0.23.0 // indirect
 	golang.org/x/oauth2 v0.12.0 // indirect
 	golang.org/x/sys v0.21.0 // indirect
 	golang.org/x/term v0.21.0 // indirect
 	golang.org/x/text v0.16.0 // indirect
-	golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect
+	golang.org/x/time v0.3.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/klog/v2 v2.70.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect
+	k8s.io/klog/v2 v2.80.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
 	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect