✨ Reimplement cache with badger

.gitignore

@@ -202,4 +202,4 @@ pkg/**/*.html
 web/.yarn/
 docs/.yarn/
 coverage.txt
-locker/
+/locker/

cmd/distribution.go

@@ -15,13 +15,16 @@
 package cmd
 
 import (
+	"context"
+
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
 	"github.com/go-sigma/sigma/pkg/cmds/distribution"
 	"github.com/go-sigma/sigma/pkg/configs"
 	"github.com/go-sigma/sigma/pkg/dal"
+	"github.com/go-sigma/sigma/pkg/dal/badger"
 	"github.com/go-sigma/sigma/pkg/inits"
 	"github.com/go-sigma/sigma/pkg/logger"
 	"github.com/go-sigma/sigma/pkg/modules/locker"
@@ -46,6 +49,12 @@ var distributionCmd = &cobra.Command{
 
 		config := ptr.To(configs.GetConfiguration())
 
+		err = badger.Initialize(context.Background(), config)
+		if err != nil {
+			log.Error().Err(err).Msg("Initialize badger with error")
+			return
+		}
+
 		err = locker.Initialize(config)
 		if err != nil {
 			log.Error().Err(err).Msg("Initialize locker with error")

cmd/server.go

@@ -15,13 +15,16 @@
 package cmd
 
 import (
+	"context"
+
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
 	"github.com/go-sigma/sigma/pkg/cmds/server"
 	"github.com/go-sigma/sigma/pkg/configs"
 	"github.com/go-sigma/sigma/pkg/dal"
+	"github.com/go-sigma/sigma/pkg/dal/badger"
 	"github.com/go-sigma/sigma/pkg/inits"
 	"github.com/go-sigma/sigma/pkg/logger"
 	"github.com/go-sigma/sigma/pkg/modules/locker"
@@ -45,6 +48,12 @@ var serverCmd = &cobra.Command{
 
 		config := ptr.To(configs.GetConfiguration())
 
+		err = badger.Initialize(context.Background(), config)
+		if err != nil {
+			log.Error().Err(err).Msg("Initialize badger with error")
+			return
+		}
+
 		err = locker.Initialize(config)
 		if err != nil {
 			log.Error().Err(err).Msg("Initialize locker with error")

cmd/worker.go

@@ -15,13 +15,16 @@
 package cmd
 
 import (
+	"context"
+
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
 	"github.com/go-sigma/sigma/pkg/cmds/worker"
 	"github.com/go-sigma/sigma/pkg/configs"
 	"github.com/go-sigma/sigma/pkg/dal"
+	"github.com/go-sigma/sigma/pkg/dal/badger"
 	"github.com/go-sigma/sigma/pkg/inits"
 	"github.com/go-sigma/sigma/pkg/logger"
 	"github.com/go-sigma/sigma/pkg/modules/locker"
@@ -45,6 +48,12 @@ var workerCmd = &cobra.Command{
 
 		config := ptr.To(configs.GetConfiguration())
 
+		err = badger.Initialize(context.Background(), config)
+		if err != nil {
+			log.Error().Err(err).Msg("Initialize badger with error")
+			return
+		}
+
 		err = locker.Initialize(config)
 		if err != nil {
 			log.Error().Err(err).Msg("Initialize locker with error")

conf/config.yaml

@@ -24,23 +24,30 @@ database:
     sslmode: disable
 
 redis:
-  # redis type available: none, external
+  # redis type available: none, external. Following all of redis config just use reference here.
   # none: means never use redis
   # external: means use the specific redis instance
   type: none
   url: redis://:sigma@localhost:6379/0
 
+badger:
+  # badger is used to implement lock and cache in a single-node mode.
+  enabled: true
+  path: /var/lib/sigma/badger/
+
 cache:
-  # the cache type available is: redis, inmemory, database
-  type: database
-  ttl: 72h
-  # please attention in multi
+  # the cache type available is: redis, inmemory, badger
+  # please attention in multi-node mode, you should use redis
+  type: badger
   inmemory:
+    prefix: sigma-cache
     size: 10240
   redis:
-  database:
-    size: 10240
-    threshold: 0.2
+    prefix: sigma-cache
+    ttl: 72h
+  badger:
+    prefix: sigma-cache
+    ttl: 72h
 
 workqueue:
   # the workqueue type available: redis, kafka, database, inmemory
@@ -53,10 +60,12 @@ workqueue:
     concurrency: 1024
 
 locker:
-  # the locker type available: redis, database
-  type: database
-  database: {}
-  redis: {}
+  # the locker type available: redis, badger
+  type: badger
+  badger:
+    prefix: sigma-locker
+  redis:
+    prefix: sigma-locker
 
 namespace:
   # push image to registry, if namespace not exist, it will be created automatically

go.mod

@@ -1,6 +1,6 @@
 module github.com/go-sigma/sigma
 
-go 1.22.0
+go 1.22.2
 
 require (
 	code.gitea.io/sdk/gitea v0.18.0
@@ -11,7 +11,7 @@ require (
 	github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible
 	github.com/anchore/syft v1.3.0
 	github.com/aquasecurity/trivy v0.50.4
-	github.com/aws/aws-sdk-go v1.51.32
+	github.com/aws/aws-sdk-go v1.52.1
 	github.com/caarlos0/env/v9 v9.0.0
 	github.com/casbin/casbin/v2 v2.88.0
 	github.com/casbin/gorm-adapter/v3 v3.24.0

go.sum

@@ -223,8 +223,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
-github.com/aws/aws-sdk-go v1.51.32 h1:A6mPui7QP4mwmovyzgtdedbRbNur1Iu0/El7hBWNHms=
-github.com/aws/aws-sdk-go v1.51.32/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
+github.com/aws/aws-sdk-go v1.52.1 h1:pYpPIuvVsawYDR0Nt3VrceizUAbtpTN3Z7xBzcZWwfI=
+github.com/aws/aws-sdk-go v1.52.1/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
 github.com/aws/aws-sdk-go-v2 v1.26.0 h1:/Ce4OCiM3EkpW7Y+xUnfAFpchU78K7/Ug01sZni9PgA=
 github.com/aws/aws-sdk-go-v2 v1.26.0/go.mod h1:35hUlJVYd+M++iLI3ALmVwMOyRYMmRqUXpTtRGW+K9I=
 github.com/aws/aws-sdk-go-v2/config v1.27.9 h1:gRx/NwpNEFSk+yQlgmk1bmxxvQ5TyJ76CWXs9XScTqg=

pkg/configs/configuration.go

@@ -38,6 +38,7 @@ type Configuration struct {
 	Database  ConfigurationDatabase  `yaml:"database"`
 	Deploy    enums.Deploy           `yaml:"deploy"`
 	Redis     ConfigurationRedis     `yaml:"redis"`
+	Badger    ConfigurationBadger    `yaml:"badger"`
 	Cache     ConfigurationCache     `yaml:"cache"`
 	WorkQueue ConfigurationWorkQueue `yaml:"workqueue"`
 	Locker    ConfigurationLocker    `yaml:"locker"`
@@ -101,28 +102,36 @@ type ConfigurationRedis struct {
 	Url  string          `yaml:"url"`
 }
 
+// ConfigurationBadger ...
+type ConfigurationBadger struct {
+	Enabled bool   `yaml:"enabled"`
+	Path    string `yaml:"path"`
+}
+
 // ConfigurationCacheRedis ...
 type ConfigurationCacheRedis struct {
+	Prefix string        `yaml:"prefix"`
+	Ttl    time.Duration `yaml:"ttl"`
 }
 
-// ConfigurationCacheInmemory ...
-type ConfigurationCacheInmemory struct {
-	Size int `yaml:"size"`
+// ConfigurationCacheBadger ...
+type ConfigurationCacheBadger struct {
+	Prefix string        `yaml:"prefix"`
+	Ttl    time.Duration `yaml:"ttl"`
 }
 
-// ConfigurationCacheDatabase ...
-type ConfigurationCacheDatabase struct {
-	Size      int64   `yaml:"size"`
-	Threshold float64 `yaml:"threshold"`
+// ConfigurationCacheInmemory ...
+type ConfigurationCacheInmemory struct {
+	Prefix string `yaml:"prefix"`
+	Size   int    `yaml:"size"`
 }
 
 // ConfigurationCache ...
 type ConfigurationCache struct {
 	Type     enums.CacherType           `yaml:"type"`
-	Ttl      time.Duration              `yaml:"ttl"`
 	Redis    ConfigurationCacheRedis    `yaml:"redis"`
 	Inmemory ConfigurationCacheInmemory `yaml:"inmemory"`
-	Database ConfigurationCacheDatabase `yaml:"database"`
+	Badger   ConfigurationCacheBadger   `yaml:"badger"`
 }
 
 type ConfigurationWorkQueueRedis struct {
@@ -148,19 +157,21 @@ type ConfigurationWorkQueue struct {
 	Inmemory ConfigurationWorkQueueInmemmory `yaml:"inmemory"`
 }
 
-// ConfigurationLockerDatabase ...
-type ConfigurationLockerDatabase struct {
-	Path string `yaml:"path"`
+// ConfigurationLockerBadger ...
+type ConfigurationLockerBadger struct {
+	Prefix string `yaml:"prefix"`
 }
 
 // ConfigurationLockerRedis ...
-type ConfigurationLockerRedis struct{}
+type ConfigurationLockerRedis struct {
+	Prefix string `yaml:"prefix"`
+}
 
 // ConfigurationLocker ...
 type ConfigurationLocker struct {
-	Type     enums.LockerType            `yaml:"type"`
-	Database ConfigurationLockerDatabase `yaml:"database"`
-	Redis    ConfigurationLockerRedis    `yaml:"redis"`
+	Type   enums.LockerType          `yaml:"type"`
+	Badger ConfigurationLockerBadger `yaml:"badger"`
+	Redis  ConfigurationLockerRedis  `yaml:"redis"`
 }
 
 // ConfigurationNamespace ...

pkg/configs/default.go

@@ -15,6 +15,7 @@
 package configs
 
 import (
+	"strings"
 	"time"
 
 	"github.com/spf13/viper"
@@ -45,9 +46,6 @@
 	if configuration.Namespace.Visibility.String() == "" {
 		configuration.Namespace.Visibility = enums.VisibilityPrivate
 	}
-	if configuration.Cache.Ttl == 0 {
-		configuration.Cache.Ttl = 72 * time.Hour
-	}
 	if configuration.Daemon.Builder.Kubernetes.Namespace == "" {
 		configuration.Daemon.Builder.Kubernetes.Namespace = "default"
 	}
@@ -57,13 +55,37 @@
 	if configuration.WorkQueue.Inmemory.Concurrency == 0 {
 		configuration.WorkQueue.Inmemory.Concurrency = 1024
 	}
-	if configuration.Cache.Inmemory.Size == 0 {
+
+	// for cache
+	if configuration.Cache.Type == enums.CacherTypeInmemory && configuration.Cache.Inmemory.Size == 0 {
 		configuration.Cache.Inmemory.Size = 10240
 	}
-	if configuration.Cache.Ttl == 0 {
-		configuration.Cache.Ttl = time.Second * 30
+	if configuration.Cache.Type == enums.CacherTypeInmemory && len(strings.TrimSpace(configuration.Cache.Inmemory.Prefix)) == 0 {
+		configuration.Cache.Inmemory.Prefix = "sigma-cache"
+	}
+	if configuration.Cache.Type == enums.CacherTypeRedis && configuration.Cache.Redis.Ttl == 0 {
+		configuration.Cache.Redis.Ttl = time.Hour * 72
+	}
+	if configuration.Cache.Type == enums.CacherTypeRedis && len(strings.TrimSpace(configuration.Cache.Redis.Prefix)) == 0 {
+		configuration.Cache.Redis.Prefix = "sigma-cache"
+	}
+	if configuration.Cache.Type == enums.CacherTypeBadger && configuration.Cache.Badger.Ttl == 0 {
+		configuration.Cache.Badger.Ttl = time.Hour * 72
+	}
+	if configuration.Cache.Type == enums.CacherTypeBadger && len(strings.TrimSpace(configuration.Cache.Badger.Prefix)) == 0 {
+		configuration.Cache.Badger.Prefix = "sigma-cache"
+	}
+
+	// for badger
+	if configuration.Badger.Enabled && len(strings.TrimSpace(configuration.Badger.Path)) == 0 {
+		configuration.Badger.Path = "/var/lib/sigma/badger/"
+	}
+
+	// for locker
+	if configuration.Locker.Type == enums.LockerTypeBadger && strings.TrimSpace(configuration.Locker.Badger.Prefix) == "" {
+		configuration.Locker.Badger.Prefix = "sigma-locker"
 	}
-	if configuration.Locker.Type == enums.LockerTypeDatabase && configuration.Locker.Database.Path == "" {
-		configuration.Locker.Database.Path = "/var/lib/sigma/badger"
+	if configuration.Locker.Type == enums.LockerTypeRedis && strings.TrimSpace(configuration.Locker.Redis.Prefix) == "" {
+		configuration.Locker.Redis.Prefix = "sigma-locker"
 	}
 }

pkg/dal/auth_test.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/go-sigma/sigma/pkg/configs"
 	"github.com/go-sigma/sigma/pkg/dal"
+	"github.com/go-sigma/sigma/pkg/dal/badger"
 	"github.com/go-sigma/sigma/pkg/dal/dao"
 	"github.com/go-sigma/sigma/pkg/dal/models"
 	"github.com/go-sigma/sigma/pkg/logger"
@@ -38,6 +39,8 @@ import (
 func TestAuth(t *testing.T) {
 	logger.SetLevel("debug")
 
+	assert.NoError(t, badger.Initialize(context.Background(), configs.Configuration{}))
+
 	err := locker.Initialize(configs.Configuration{})
 	assert.NoError(t, err)