Skip to content

Commit

Permalink
✨ Reimplement cache with badger
Browse files Browse the repository at this point in the history
  • Loading branch information
tosone committed May 4, 2024
1 parent 6c643f6 commit eaabf0b
Show file tree
Hide file tree
Showing 36 changed files with 299 additions and 1,212 deletions.
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -202,4 +202,4 @@ pkg/**/*.html
web/.yarn/
docs/.yarn/
coverage.txt
locker/
/locker/
9 changes: 9 additions & 0 deletions cmd/distribution.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,16 @@
package cmd

import (
"context"

"github.com/rs/zerolog/log"
"github.com/spf13/cobra"
"github.com/spf13/viper"

"github.com/go-sigma/sigma/pkg/cmds/distribution"
"github.com/go-sigma/sigma/pkg/configs"
"github.com/go-sigma/sigma/pkg/dal"
"github.com/go-sigma/sigma/pkg/dal/badger"
"github.com/go-sigma/sigma/pkg/inits"
"github.com/go-sigma/sigma/pkg/logger"
"github.com/go-sigma/sigma/pkg/modules/locker"
Expand All @@ -46,6 +49,12 @@ var distributionCmd = &cobra.Command{

config := ptr.To(configs.GetConfiguration())

err = badger.Initialize(context.Background(), config)
if err != nil {
log.Error().Err(err).Msg("Initialize badger with error")
return
}

err = locker.Initialize(config)
if err != nil {
log.Error().Err(err).Msg("Initialize locker with error")
Expand Down
9 changes: 9 additions & 0 deletions cmd/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,16 @@
package cmd

import (
"context"

"github.com/rs/zerolog/log"
"github.com/spf13/cobra"
"github.com/spf13/viper"

"github.com/go-sigma/sigma/pkg/cmds/server"
"github.com/go-sigma/sigma/pkg/configs"
"github.com/go-sigma/sigma/pkg/dal"
"github.com/go-sigma/sigma/pkg/dal/badger"
"github.com/go-sigma/sigma/pkg/inits"
"github.com/go-sigma/sigma/pkg/logger"
"github.com/go-sigma/sigma/pkg/modules/locker"
Expand All @@ -45,6 +48,12 @@ var serverCmd = &cobra.Command{

config := ptr.To(configs.GetConfiguration())

err = badger.Initialize(context.Background(), config)
if err != nil {
log.Error().Err(err).Msg("Initialize badger with error")
return
}

err = locker.Initialize(config)
if err != nil {
log.Error().Err(err).Msg("Initialize locker with error")
Expand Down
9 changes: 9 additions & 0 deletions cmd/worker.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,16 @@
package cmd

import (
"context"

"github.com/rs/zerolog/log"
"github.com/spf13/cobra"
"github.com/spf13/viper"

"github.com/go-sigma/sigma/pkg/cmds/worker"
"github.com/go-sigma/sigma/pkg/configs"
"github.com/go-sigma/sigma/pkg/dal"
"github.com/go-sigma/sigma/pkg/dal/badger"
"github.com/go-sigma/sigma/pkg/inits"
"github.com/go-sigma/sigma/pkg/logger"
"github.com/go-sigma/sigma/pkg/modules/locker"
Expand All @@ -45,6 +48,12 @@ var workerCmd = &cobra.Command{

config := ptr.To(configs.GetConfiguration())

err = badger.Initialize(context.Background(), config)
if err != nil {
log.Error().Err(err).Msg("Initialize badger with error")
return
}

err = locker.Initialize(config)
if err != nil {
log.Error().Err(err).Msg("Initialize locker with error")
Expand Down
33 changes: 21 additions & 12 deletions conf/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -24,23 +24,30 @@ database:
sslmode: disable

redis:
# redis type available: none, external
# redis type available: none, external. Following all of redis config just use reference here.
# none: means never use redis
# external: means use the specific redis instance
type: none
url: redis://:sigma@localhost:6379/0

badger:
# badger is used to implement lock and cache in a single-node mode.
enabled: true
path: /var/lib/sigma/badger/

cache:
# the cache type available is: redis, inmemory, database
type: database
ttl: 72h
# please attention in multi
# the cache type available is: redis, inmemory, badger
# please attention in multi-node mode, you should use redis
type: badger
inmemory:
prefix: sigma-cache
size: 10240
redis:
database:
size: 10240
threshold: 0.2
prefix: sigma-cache
ttl: 72h
badger:
prefix: sigma-cache
ttl: 72h

workqueue:
# the workqueue type available: redis, kafka, database, inmemory
Expand All @@ -53,10 +60,12 @@ workqueue:
concurrency: 1024

locker:
# the locker type available: redis, database
type: database
database: {}
redis: {}
# the locker type available: redis, badger
type: badger
badger:
prefix: sigma-locker
redis:
prefix: sigma-locker

namespace:
# push image to registry, if namespace not exist, it will be created automatically
Expand Down
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/go-sigma/sigma

go 1.22.0
go 1.22.2

require (
code.gitea.io/sdk/gitea v0.18.0
Expand All @@ -11,7 +11,7 @@ require (
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible
github.com/anchore/syft v1.3.0
github.com/aquasecurity/trivy v0.50.4
github.com/aws/aws-sdk-go v1.51.32
github.com/aws/aws-sdk-go v1.52.1
github.com/caarlos0/env/v9 v9.0.0
github.com/casbin/casbin/v2 v2.88.0
github.com/casbin/gorm-adapter/v3 v3.24.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -223,8 +223,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
github.com/aws/aws-sdk-go v1.51.32 h1:A6mPui7QP4mwmovyzgtdedbRbNur1Iu0/El7hBWNHms=
github.com/aws/aws-sdk-go v1.51.32/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
github.com/aws/aws-sdk-go v1.52.1 h1:pYpPIuvVsawYDR0Nt3VrceizUAbtpTN3Z7xBzcZWwfI=
github.com/aws/aws-sdk-go v1.52.1/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
github.com/aws/aws-sdk-go-v2 v1.26.0 h1:/Ce4OCiM3EkpW7Y+xUnfAFpchU78K7/Ug01sZni9PgA=
github.com/aws/aws-sdk-go-v2 v1.26.0/go.mod h1:35hUlJVYd+M++iLI3ALmVwMOyRYMmRqUXpTtRGW+K9I=
github.com/aws/aws-sdk-go-v2/config v1.27.9 h1:gRx/NwpNEFSk+yQlgmk1bmxxvQ5TyJ76CWXs9XScTqg=
Expand Down
43 changes: 27 additions & 16 deletions pkg/configs/configuration.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ type Configuration struct {
Database ConfigurationDatabase `yaml:"database"`
Deploy enums.Deploy `yaml:"deploy"`
Redis ConfigurationRedis `yaml:"redis"`
Badger ConfigurationBadger `yaml:"badger"`
Cache ConfigurationCache `yaml:"cache"`
WorkQueue ConfigurationWorkQueue `yaml:"workqueue"`
Locker ConfigurationLocker `yaml:"locker"`
Expand Down Expand Up @@ -101,28 +102,36 @@ type ConfigurationRedis struct {
Url string `yaml:"url"`
}

// ConfigurationBadger ...
type ConfigurationBadger struct {
Enabled bool `yaml:"enabled"`
Path string `yaml:"path"`
}

// ConfigurationCacheRedis ...
type ConfigurationCacheRedis struct {
Prefix string `yaml:"prefix"`
Ttl time.Duration `yaml:"ttl"`
}

// ConfigurationCacheInmemory ...
type ConfigurationCacheInmemory struct {
Size int `yaml:"size"`
// ConfigurationCacheBadger ...
type ConfigurationCacheBadger struct {
Prefix string `yaml:"prefix"`
Ttl time.Duration `yaml:"ttl"`
}

// ConfigurationCacheDatabase ...
type ConfigurationCacheDatabase struct {
Size int64 `yaml:"size"`
Threshold float64 `yaml:"threshold"`
// ConfigurationCacheInmemory ...
type ConfigurationCacheInmemory struct {
Prefix string `yaml:"prefix"`
Size int `yaml:"size"`
}

// ConfigurationCache ...
type ConfigurationCache struct {
Type enums.CacherType `yaml:"type"`
Ttl time.Duration `yaml:"ttl"`
Redis ConfigurationCacheRedis `yaml:"redis"`
Inmemory ConfigurationCacheInmemory `yaml:"inmemory"`
Database ConfigurationCacheDatabase `yaml:"database"`
Badger ConfigurationCacheBadger `yaml:"badger"`
}

type ConfigurationWorkQueueRedis struct {
Expand All @@ -148,19 +157,21 @@ type ConfigurationWorkQueue struct {
Inmemory ConfigurationWorkQueueInmemmory `yaml:"inmemory"`
}

// ConfigurationLockerDatabase ...
type ConfigurationLockerDatabase struct {
Path string `yaml:"path"`
// ConfigurationLockerBadger ...
type ConfigurationLockerBadger struct {
Prefix string `yaml:"prefix"`
}

// ConfigurationLockerRedis ...
type ConfigurationLockerRedis struct{}
type ConfigurationLockerRedis struct {
Prefix string `yaml:"prefix"`
}

// ConfigurationLocker ...
type ConfigurationLocker struct {
Type enums.LockerType `yaml:"type"`
Database ConfigurationLockerDatabase `yaml:"database"`
Redis ConfigurationLockerRedis `yaml:"redis"`
Type enums.LockerType `yaml:"type"`
Badger ConfigurationLockerBadger `yaml:"badger"`
Redis ConfigurationLockerRedis `yaml:"redis"`
}

// ConfigurationNamespace ...
Expand Down
38 changes: 30 additions & 8 deletions pkg/configs/default.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package configs

import (
"strings"
"time"

"github.com/spf13/viper"
Expand Down Expand Up @@ -45,9 +46,6 @@ func defaultSettings() {
if configuration.Namespace.Visibility.String() == "" {
configuration.Namespace.Visibility = enums.VisibilityPrivate
}
if configuration.Cache.Ttl == 0 {
configuration.Cache.Ttl = 72 * time.Hour
}
if configuration.Daemon.Builder.Kubernetes.Namespace == "" {
configuration.Daemon.Builder.Kubernetes.Namespace = "default"
}
Expand All @@ -57,13 +55,37 @@ func defaultSettings() {
if configuration.WorkQueue.Inmemory.Concurrency == 0 {
configuration.WorkQueue.Inmemory.Concurrency = 1024
}
if configuration.Cache.Inmemory.Size == 0 {

// for cache
if configuration.Cache.Type == enums.CacherTypeInmemory && configuration.Cache.Inmemory.Size == 0 {
configuration.Cache.Inmemory.Size = 10240
}
if configuration.Cache.Ttl == 0 {
configuration.Cache.Ttl = time.Second * 30
if configuration.Cache.Type == enums.CacherTypeInmemory && len(strings.TrimSpace(configuration.Cache.Inmemory.Prefix)) == 0 {
configuration.Cache.Inmemory.Prefix = "sigma-cache"
}

Check warning on line 65 in pkg/configs/default.go

View check run for this annotation

Codecov / codecov/patch

pkg/configs/default.go#L64-L65

Added lines #L64 - L65 were not covered by tests
if configuration.Cache.Type == enums.CacherTypeRedis && configuration.Cache.Redis.Ttl == 0 {
configuration.Cache.Redis.Ttl = time.Hour * 72
}

Check warning on line 68 in pkg/configs/default.go

View check run for this annotation

Codecov / codecov/patch

pkg/configs/default.go#L67-L68

Added lines #L67 - L68 were not covered by tests
if configuration.Cache.Type == enums.CacherTypeRedis && len(strings.TrimSpace(configuration.Cache.Redis.Prefix)) == 0 {
configuration.Cache.Redis.Prefix = "sigma-cache"
}

Check warning on line 71 in pkg/configs/default.go

View check run for this annotation

Codecov / codecov/patch

pkg/configs/default.go#L70-L71

Added lines #L70 - L71 were not covered by tests
if configuration.Cache.Type == enums.CacherTypeBadger && configuration.Cache.Badger.Ttl == 0 {
configuration.Cache.Badger.Ttl = time.Hour * 72
}

Check warning on line 74 in pkg/configs/default.go

View check run for this annotation

Codecov / codecov/patch

pkg/configs/default.go#L73-L74

Added lines #L73 - L74 were not covered by tests
if configuration.Cache.Type == enums.CacherTypeBadger && len(strings.TrimSpace(configuration.Cache.Badger.Prefix)) == 0 {
configuration.Cache.Badger.Prefix = "sigma-cache"
}

Check warning on line 77 in pkg/configs/default.go

View check run for this annotation

Codecov / codecov/patch

pkg/configs/default.go#L76-L77

Added lines #L76 - L77 were not covered by tests

// for badger
if configuration.Badger.Enabled && len(strings.TrimSpace(configuration.Badger.Path)) == 0 {
configuration.Badger.Path = "/var/lib/sigma/badger/"
}

Check warning on line 82 in pkg/configs/default.go

View check run for this annotation

Codecov / codecov/patch

pkg/configs/default.go#L81-L82

Added lines #L81 - L82 were not covered by tests

// for locker
if configuration.Locker.Type == enums.LockerTypeBadger && strings.TrimSpace(configuration.Locker.Badger.Prefix) == "" {
configuration.Locker.Badger.Prefix = "sigma-locker"

Check warning on line 86 in pkg/configs/default.go

View check run for this annotation

Codecov / codecov/patch

pkg/configs/default.go#L86

Added line #L86 was not covered by tests
}
if configuration.Locker.Type == enums.LockerTypeDatabase && configuration.Locker.Database.Path == "" {
configuration.Locker.Database.Path = "/var/lib/sigma/badger"
if configuration.Locker.Type == enums.LockerTypeRedis && strings.TrimSpace(configuration.Locker.Redis.Prefix) == "" {
configuration.Locker.Redis.Prefix = "sigma-locker"

Check warning on line 89 in pkg/configs/default.go

View check run for this annotation

Codecov / codecov/patch

pkg/configs/default.go#L89

Added line #L89 was not covered by tests
}
}
3 changes: 3 additions & 0 deletions pkg/dal/auth_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (

"github.com/go-sigma/sigma/pkg/configs"
"github.com/go-sigma/sigma/pkg/dal"
"github.com/go-sigma/sigma/pkg/dal/badger"
"github.com/go-sigma/sigma/pkg/dal/dao"
"github.com/go-sigma/sigma/pkg/dal/models"
"github.com/go-sigma/sigma/pkg/logger"
Expand All @@ -38,6 +39,8 @@ import (
func TestAuth(t *testing.T) {
logger.SetLevel("debug")

assert.NoError(t, badger.Initialize(context.Background(), configs.Configuration{}))

err := locker.Initialize(configs.Configuration{})
assert.NoError(t, err)

Expand Down
Loading

0 comments on commit eaabf0b

Please sign in to comment.