add: redis cache for remote images

config.json

@@ -10,5 +10,7 @@
   "ENABLE_EXTRA_PARAMS": false,
   "READ_BUFFER_SIZE": 4096,
   "CONCURRENCY": 262144,
-  "DISABLE_KEEPALIVE": false
+  "DISABLE_KEEPALIVE": false,
+  "REDIS_URL": "",
+  "REDIS_TTL": 259200
 }

config/config.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"context"
 	"encoding/json"
 	"flag"
 	"os"
@@ -11,6 +12,7 @@ import (
 	"time"
 
 	"github.com/patrickmn/go-cache"
+	"github.com/redis/go-redis/v9"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -33,7 +35,9 @@ const (
   "ENABLE_EXTRA_PARAMS": false
   "READ_BUFFER_SIZE": 4096,
   "CONCURRENCY": 262144,
-  "DISABLE_KEEPALIVE": false
+  "DISABLE_KEEPALIVE": false,
+  "REDIS_URL": "",
+  "REDIS_TTL": 259200,
 }`
 )
 
@@ -52,6 +56,7 @@ var (
 	RemoteRaw      = "./remote-raw"
 	Metadata       = "./metadata"
 	LocalHostAlias = "local"
+	Cache          *redis.Client
 )
 
 type MetaFile struct {
@@ -73,6 +78,8 @@ type WebpConfig struct {
 	ReadBufferSize    int               `json:"READ_BUFFER_SIZE"`
 	Concurrency       int               `json:"CONCURRENCY"`
 	DisableKeepalive  bool              `json:"DISABLE_KEEPALIVE"`
+	RedisURL          string            `json:"REDIS_URL"`
+	RedisTTL     	  int               `json:"REDIS_TTL"`
 }
 
 func NewWebPConfig() *WebpConfig {
@@ -89,6 +96,8 @@ func NewWebPConfig() *WebpConfig {
 		ReadBufferSize:    4096,
 		Concurrency:       262144,
 		DisableKeepalive:  false,
+		RedisURL:          "",
+		RedisTTL:          259200,
 	}
 }
 
@@ -184,6 +193,27 @@ func LoadConfig() {
 			log.Warnf("WEBP_DISABLE_KEEPALIVE is not a valid boolean, using value in config.json %t", Config.DisableKeepalive)
 		}
 	}
+	if os.Getenv("REDIS_URL") != "" {
+		Config.RedisURL = os.Getenv("REDIS_URL")
+	}
+	if os.Getenv("REDIS_TTL") != "" {
+		redisTTL, err := strconv.Atoi(os.Getenv("REDIS_TTL"))
+		if err != nil {
+			log.Warnf("REDIS_TTL is not a valid integer, using value in config.json %d", Config.RedisTTL)
+		} else {
+			Config.RedisTTL = redisTTL
+		}
+	}
+
+	opts, err := redis.ParseURL(Config.RedisURL)
+	if err != nil {
+		log.Warnf("Redis URL: %v cannot be parsed", Config.RedisURL)
+	} else {
+		Cache = redis.NewClient(opts)
+		if err := Cache.Ping(context.TODO()).Err(); err != nil {
+			log.Warnf("Pinging redis db failed with exception: %v", err.Error())
+		}
+	}
 
 	log.Debugln("Config init complete")
 	log.Debugln("Config", Config)

config/config_test.go

@@ -21,6 +21,8 @@ func TestLoadConfig(t *testing.T) {
 	assert.Equal(t, Config.ImgPath, "./pics")
 	assert.Equal(t, Config.ImageMap, map[string]string{})
 	assert.Equal(t, Config.ExhaustPath, "./exhaust")
+	assert.Equal(t, Config.RedisURL, "")
+	assert.Equal(t, Config.RedisTTL, 259200)
 }
 
 func TestSwitchProxyMode(t *testing.T) {
@@ -43,7 +45,7 @@ func TestParseImgMap(t *testing.T) {
 		"httpx://example.com": "../pics",
 		"ftp://example.com": "../pics",
 	}
-	
+
 	assert.Equal(t, empty, parseImgMap(empty))
 	assert.Equal(t, empty, parseImgMap(bad))
 	assert.Equal(t, good, parseImgMap(good))

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c
 	github.com/jeremytorres/rawparser v1.0.2
 	github.com/patrickmn/go-cache v2.1.0+incompatible
+	github.com/redis/go-redis/v9 v9.4.0
 	github.com/schollz/progressbar/v3 v3.14.1
 	github.com/sirupsen/logrus v1.9.3
 	github.com/stretchr/testify v1.8.4
@@ -18,7 +19,9 @@ require (
 
 require (
 	github.com/andybalholm/brotli v1.0.5 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/google/uuid v1.4.0 // indirect
 	github.com/klauspost/compress v1.17.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect

go.sum

@@ -2,13 +2,21 @@ github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
 github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
+github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
+github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
+github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
+github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davidbyttow/govips/v2 v2.13.0 h1:5MK9ZcXZC5GzUR9Ca8fJwOYqMgll/H096ec0PJP59QM=
 github.com/davidbyttow/govips/v2 v2.13.0/go.mod h1:LPTrwWtNa5n4yl9UC52YBOEGdZcY5hDTP4Ms2QWasTw=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/gofiber/fiber/v2 v2.51.0 h1:JNACcZy5e2tGApWB2QrRpenTWn0fq0hkFm6k0C86gKQ=
 github.com/gofiber/fiber/v2 v2.51.0/go.mod h1:xaQRZQJGqnKOQnbQw+ltvku3/h8QxvNi8o6JiJ7Ll0U=
 github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
@@ -40,6 +48,8 @@ github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaR
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/redis/go-redis/v9 v9.4.0 h1:Yzoz33UZw9I/mFhx4MNrB6Fk+XHO1VukNcCa1+lwyKk=
+github.com/redis/go-redis/v9 v9.4.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
 github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=

handler/remote.go

@@ -2,16 +2,19 @@ package handler
 
 import (
 	"bytes"
+	"context"
 	"net/http"
 	"os"
 	"path"
 	"path/filepath"
 	"strings"
+	"time"
 	"webp_server_go/config"
 	"webp_server_go/helper"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/h2non/filetype"
+	"github.com/redis/go-redis/v9"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -78,8 +81,33 @@ func downloadFile(filepath string, url string) {
 func fetchRemoteImg(url string, subdir string) config.MetaFile {
 	// url is https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200
 	// How do we know if the remote img is changed? we're using hash(etag+length)
-	log.Infof("Remote Addr is %s, pinging for info...", url)
-	etag := pingURL(url)
+	var cacheKey, etag string
+	if config.Cache != nil {
+		cacheKey = subdir+":"+helper.HashString(url)
+	}
+
+	if cacheKey != "" {
+		val, err := config.Cache.Get(context.TODO(), cacheKey).Result()
+		if err == nil {
+			log.Infof("Using cache for remote addr: %s", url)
+			etag = val
+		} else if err != redis.Nil {
+			log.Warnf("Redis server unreachable, skipping for now.")
+		}
+	}
+
+	if etag == "" {
+		log.Infof("Remote Addr is %s, pinging for info...", url)
+		etag = pingURL(url)
+		if cacheKey != "" {
+			err := config.Cache.Set(context.TODO(), cacheKey, etag, time.Duration(config.Config.RedisTTL) * time.Second).Err()
+			if err != nil {
+				log.Warnf("Redis server unreachable, skipping for now.")
+			}
+		}
+
+	}
+
 	metadata := helper.ReadMetadata(url, etag, subdir)
 	localRawImagePath := path.Join(config.RemoteRaw, subdir, metadata.Id)