From 43c275e3ec2ebcb9b7f60dc85be1422d111312b2 Mon Sep 17 00:00:00 2001 From: Nova Kwok Date: Fri, 22 Mar 2024 15:12:09 +0800 Subject: [PATCH] Adds JPEG XL support, max_height/max_width support (#321) * WIP JXL * Fix test * Tries to fix autobuild * Tries to fix autobuild * Add setup go in codeql * Bump actions version * Do not print curl output in CI * Do not print curl output in CI * Remove Metadata on RAW image * Update sample config * better loop * Prefetch should also respect AllowedType * Better Export params and UA handle * Only do conversion on supported formats * CONVERT_TYPES default to webp only * CONVERT_TYPES default to webp only * Add GIF to AllowedTypes * Update README --- .github/workflows/CI.yaml | 2 +- .github/workflows/codeql-analysis.yml | 26 +---- .github/workflows/integration-test.yaml | 4 +- .github/workflows/release_docker_image.yaml | 4 +- README.md | 6 +- config.json | 7 +- config/config.go | 114 ++++++++++++++------ encoder/encoder.go | 70 ++++++++++-- encoder/prefetch.go | 21 +++- encoder/process.go | 60 ++++++++++- encoder/process_test.go | 90 ++++++++++++++++ go.mod | 2 + go.sum | 4 +- handler/router.go | 34 +++--- handler/router_test.go | 13 ++- helper/helper.go | 27 +++-- helper/helper_test.go | 31 ++++-- helper/metadata.go | 6 +- helper/metadata_test.go | 4 +- webp-server.go | 6 +- 20 files changed, 405 insertions(+), 126 deletions(-) create mode 100644 encoder/process_test.go diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 9196a223f..6929a7367 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -40,7 +40,7 @@ jobs: submodules: true - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Cache Docker layers uses: actions/cache@v2 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index edbca9b2e..d4fe42e0f 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -15,7 +15,6 @@ on: push: branches: [ master ] pull_request: - # The branches below must be a subset of the branches above branches: [ master ] schedule: - cron: '32 20 * * 2' @@ -33,38 +32,23 @@ jobs: fail-fast: false matrix: language: [ 'go' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] - # Learn more about CodeQL language support at https://git.io/codeql-language-support steps: - name: Checkout repository uses: actions/checkout@v3 - # Initializes the CodeQL tools for scanning. + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version: '1.22' + - name: Initialize CodeQL uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild uses: github/codeql-action/autobuild@v2 - # ℹ️ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - - # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml index 68328c98f..ae5ff2f0c 100644 --- a/.github/workflows/integration-test.yaml +++ b/.github/workflows/integration-test.yaml @@ -32,7 +32,7 @@ jobs: - name: Send Requests to Server run: | cd pics - find * -type f -print | xargs -I {} curl -H "Accept: image/webp" http://localhost:3333/{} + find * -type f -print | xargs -I {} curl -o /dev/null -H "Accept: image/webp" http://localhost:3333/{} - name: Get container RAM stats run: | @@ -61,7 +61,7 @@ jobs: - name: Send Requests to Server run: | cd pics - find * -type f -print | xargs -I {} curl -H "Accept: image/webp" http://localhost:3333/{} + find * -type f -print | xargs -I {} curl -o /dev/null -H "Accept: image/webp" http://localhost:3333/{} - name: Get container RAM stats run: | diff --git a/.github/workflows/release_docker_image.yaml b/.github/workflows/release_docker_image.yaml index b4e773f3c..dad0d1d20 100644 --- a/.github/workflows/release_docker_image.yaml +++ b/.github/workflows/release_docker_image.yaml @@ -43,10 +43,10 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 - name: Cache Docker layers uses: actions/cache@v2 diff --git a/README.md b/README.md index 52f157411..2b3a91811 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Currently supported image format: JPEG, PNG, BMP, GIF, SVG, HEIC, NEF, WEBP > e.g When you visit `https://your.website/pics/tsuki.jpg`，it will serve as `image/webp`/`image/avif` format without changing the URL. > -> GIF image will not be converted to AVIF format even with `ENABLE_AVIF` to `true`, because the converted AVIF image is not animated. +> GIF image will not be converted to AVIF format because the converted AVIF image is not animated. ## Usage with Docker(recommended) @@ -32,8 +32,6 @@ services: image: webpsh/webp-server-go # image: ghcr.io/webp-sh/webp_server_go restart: always - environment: - - MALLOC_ARENA_MAX=1 volumes: - ./path/to/pics:/opt/pics - ./exhaust:/opt/exhaust @@ -74,8 +72,6 @@ services: image: webpsh/webp-server-go # image: ghcr.io/webp-sh/webp_server_go restart: always - environment: - - MALLOC_ARENA_MAX=1 volumes: - ./path/to/pics:/opt/pics - ./path/to/exhaust:/opt/exhaust diff --git a/config.json b/config.json index 8bad6e4c2..bb3e051b5 100644 --- a/config.json +++ b/config.json @@ -4,12 +4,13 @@ "QUALITY": "80", "IMG_PATH": "./pics", "EXHAUST_PATH": "./exhaust", - "ALLOWED_TYPES": ["jpg","png","jpeg","bmp","gif","svg","heic"], "IMG_MAP": {}, - "ENABLE_AVIF": false, + "ALLOWED_TYPES": ["jpg","png","jpeg","gif","bmp","svg","heic","nef"], + "CONVERT_TYPES": ["webp"], + "STRIP_METADATA": true, "ENABLE_EXTRA_PARAMS": false, "READ_BUFFER_SIZE": 4096, "CONCURRENCY": 262144, "DISABLE_KEEPALIVE": false, "CACHE_TTL": 259200 -} +} \ No newline at end of file diff --git a/config/config.go b/config/config.go index 4532df39e..3217f96f6 100644 --- a/config/config.go +++ b/config/config.go @@ -6,6 +6,7 @@ import ( "os" "regexp" "runtime" + "slices" "strconv" "strings" "time" @@ -28,13 +29,14 @@ const ( "IMG_PATH": "./pics", "EXHAUST_PATH": "./exhaust", "IMG_MAP": {}, - "ALLOWED_TYPES": ["jpg","png","jpeg","bmp","svg","heic","nef"], - "ENABLE_AVIF": false, - "ENABLE_EXTRA_PARAMS": false + "ALLOWED_TYPES": ["jpg","png","jpeg","gif","bmp","svg","heic","nef"], + "CONVERT_TYPES": ["webp"], + "STRIP_METADATA": true, + "ENABLE_EXTRA_PARAMS": false, "READ_BUFFER_SIZE": 4096, "CONCURRENCY": 262144, "DISABLE_KEEPALIVE": false, - "CACHE_TTL": 259200, + "CACHE_TTL": 259200 }` ) @@ -47,7 +49,7 @@ var ( ProxyMode bool Prefetch bool Config = NewWebPConfig() - Version = "0.10.8" + Version = "0.11.0" WriteLock = cache.New(5*time.Minute, 10*time.Minute) ConvertLock = cache.New(5*time.Minute, 10*time.Minute) RemoteRaw = "./remote-raw" @@ -63,32 +65,44 @@ type MetaFile struct { } type WebpConfig struct { - Host string `json:"HOST"` - Port string `json:"PORT"` - ImgPath string `json:"IMG_PATH"` - Quality int `json:"QUALITY,string"` - AllowedTypes []string `json:"ALLOWED_TYPES"` - ImageMap map[string]string `json:"IMG_MAP"` - ExhaustPath string `json:"EXHAUST_PATH"` - EnableAVIF bool `json:"ENABLE_AVIF"` - EnableExtraParams bool `json:"ENABLE_EXTRA_PARAMS"` - ReadBufferSize int `json:"READ_BUFFER_SIZE"` - Concurrency int `json:"CONCURRENCY"` - DisableKeepalive bool `json:"DISABLE_KEEPALIVE"` - CacheTTL int `json:"CACHE_TTL"` + Host string `json:"HOST"` + Port string `json:"PORT"` + ImgPath string `json:"IMG_PATH"` + Quality int `json:"QUALITY,string"` + AllowedTypes []string `json:"ALLOWED_TYPES"` + ConvertTypes []string `json:"CONVERT_TYPES"` + ImageMap map[string]string `json:"IMG_MAP"` + ExhaustPath string `json:"EXHAUST_PATH"` + + EnableWebP bool `json:"ENABLE_WEBP"` + EnableAVIF bool `json:"ENABLE_AVIF"` + EnableJXL bool `json:"ENABLE_JXL"` + + EnableExtraParams bool `json:"ENABLE_EXTRA_PARAMS"` + StripMetadata bool `json:"STRIP_METADATA"` + ReadBufferSize int `json:"READ_BUFFER_SIZE"` + Concurrency int `json:"CONCURRENCY"` + DisableKeepalive bool `json:"DISABLE_KEEPALIVE"` + CacheTTL int `json:"CACHE_TTL"` } func NewWebPConfig() *WebpConfig { return &WebpConfig{ - Host: "0.0.0.0", - Port: "3333", - ImgPath: "./pics", - Quality: 80, - AllowedTypes: []string{"jpg", "png", "jpeg", "bmp", "svg", "nef", "heic", "webp"}, - ImageMap: map[string]string{}, - ExhaustPath: "./exhaust", - EnableAVIF: false, + Host: "0.0.0.0", + Port: "3333", + ImgPath: "./pics", + Quality: 80, + AllowedTypes: []string{"jpg", "png", "jpeg", "bmp", "gif", "svg", "nef", "heic", "webp"}, + ConvertTypes: []string{"webp"}, + ImageMap: map[string]string{}, + ExhaustPath: "./exhaust", + + EnableWebP: false, + EnableAVIF: false, + EnableJXL: false, + EnableExtraParams: false, + StripMetadata: true, ReadBufferSize: 4096, Concurrency: 262144, DisableKeepalive: false, @@ -115,6 +129,16 @@ func LoadConfig() { switchProxyMode() Config.ImageMap = parseImgMap(Config.ImageMap) + if slices.Contains(Config.ConvertTypes, "webp") { + Config.EnableWebP = true + } + if slices.Contains(Config.ConvertTypes, "avif") { + Config.EnableAVIF = true + } + if slices.Contains(Config.ConvertTypes, "jxl") { + Config.EnableJXL = true + } + // Read from ENV for override if os.Getenv("WEBP_HOST") != "" { Config.Host = os.Getenv("WEBP_HOST") @@ -139,16 +163,24 @@ func LoadConfig() { if os.Getenv("WEBP_ALLOWED_TYPES") != "" { Config.AllowedTypes = strings.Split(os.Getenv("WEBP_ALLOWED_TYPES"), ",") } - if os.Getenv("WEBP_ENABLE_AVIF") != "" { - enableAVIF := os.Getenv("WEBP_ENABLE_AVIF") - if enableAVIF == "true" { + + // Override enabled convert types + if os.Getenv("WEBP_CONVERT_TYPES") != "" { + Config.ConvertTypes = strings.Split(os.Getenv("WEBP_CONVERT_TYPES"), ",") + Config.EnableWebP = false + Config.EnableAVIF = false + Config.EnableJXL = false + if slices.Contains(Config.ConvertTypes, "webp") { + Config.EnableWebP = true + } + if slices.Contains(Config.ConvertTypes, "avif") { Config.EnableAVIF = true - } else if enableAVIF == "false" { - Config.EnableAVIF = false - } else { - log.Warnf("WEBP_ENABLE_AVIF is not a valid boolean, using value in config.json %t", Config.EnableAVIF) + } + if slices.Contains(Config.ConvertTypes, "jxl") { + Config.EnableJXL = true } } + if os.Getenv("WEBP_ENABLE_EXTRA_PARAMS") != "" { enableExtraParams := os.Getenv("WEBP_ENABLE_EXTRA_PARAMS") if enableExtraParams == "true" { @@ -159,6 +191,16 @@ func LoadConfig() { log.Warnf("WEBP_ENABLE_EXTRA_PARAMS is not a valid boolean, using value in config.json %t", Config.EnableExtraParams) } } + if os.Getenv("WEBP_STRIP_METADATA") != "" { + stripMetadata := os.Getenv("WEBP_STRIP_METADATA") + if stripMetadata == "true" { + Config.StripMetadata = true + } else if stripMetadata == "false" { + Config.StripMetadata = false + } else { + log.Warnf("WEBP_STRIP_METADATA is not a valid boolean, using value in config.json %t", Config.StripMetadata) + } + } if os.Getenv("WEBP_IMG_MAP") != "" { // TODO } @@ -223,8 +265,10 @@ func parseImgMap(imgMap map[string]string) map[string]string { } type ExtraParams struct { - Width int // in px - Height int // in px + Width int // in px + Height int // in px + MaxWidth int // in px + MaxHeight int // in px } func switchProxyMode() { diff --git a/encoder/encoder.go b/encoder/encoder.go index 1d28d7f66..7b1e223f9 100644 --- a/encoder/encoder.go +++ b/encoder/encoder.go @@ -33,7 +33,7 @@ func init() { intMinusOne.Set(-1) } -func ConvertFilter(rawPath, avifPath, webpPath string, extraParams config.ExtraParams, c chan int) { +func ConvertFilter(rawPath, jxlPath, avifPath, webpPath string, extraParams config.ExtraParams, supportedFormats map[string]bool, c chan int) { // Wait for the conversion to complete and return the converted image retryDelay := 100 * time.Millisecond // Initial retry delay @@ -53,8 +53,8 @@ func ConvertFilter(rawPath, avifPath, webpPath string, extraParams config.ExtraP defer config.ConvertLock.Delete(rawPath) var wg sync.WaitGroup - wg.Add(2) - if !helper.ImageExists(avifPath) && config.Config.EnableAVIF { + wg.Add(3) + if !helper.ImageExists(avifPath) && config.Config.EnableAVIF && supportedFormats["avif"] { go func() { err := convertImage(rawPath, avifPath, "avif", extraParams) if err != nil { @@ -66,7 +66,7 @@ func ConvertFilter(rawPath, avifPath, webpPath string, extraParams config.ExtraP wg.Done() } - if !helper.ImageExists(webpPath) { + if !helper.ImageExists(webpPath) && config.Config.EnableWebP && supportedFormats["webp"] { go func() { err := convertImage(rawPath, webpPath, "webp", extraParams) if err != nil { @@ -77,6 +77,19 @@ func ConvertFilter(rawPath, avifPath, webpPath string, extraParams config.ExtraP } else { wg.Done() } + + if !helper.ImageExists(jxlPath) && config.Config.EnableJXL && supportedFormats["jxl"] { + go func() { + err := convertImage(rawPath, jxlPath, "jxl", extraParams) + if err != nil { + log.Errorln(err) + } + defer wg.Done() + }() + } else { + wg.Done() + } + wg.Wait() if c != nil { @@ -123,11 +136,52 @@ func convertImage(rawPath, optimizedPath, imageType string, extraParams config.E err = webpEncoder(img, rawPath, optimizedPath) case "avif": err = avifEncoder(img, rawPath, optimizedPath) + case "jxl": + err = jxlEncoder(img, rawPath, optimizedPath) } return err } +func jxlEncoder(img *vips.ImageRef, rawPath string, optimizedPath string) error { + var ( + buf []byte + quality = config.Config.Quality + err error + ) + + // If quality >= 100, we use lossless mode + if quality >= 100 { + buf, _, err = img.ExportJxl(&vips.JxlExportParams{ + Effort: 1, + Tier: 4, + Lossless: true, + Distance: 1.0, + }) + } else { + buf, _, err = img.ExportJxl(&vips.JxlExportParams{ + Effort: 1, + Tier: 4, + Quality: quality, + Lossless: false, + Distance: 1.0, + }) + } + + if err != nil { + log.Warnf("Can't encode source image: %v to JXL", err) + return err + } + + if err := os.WriteFile(optimizedPath, buf, 0600); err != nil { + log.Error(err) + return err + } + + convertLog("JXL", rawPath, optimizedPath, quality) + return nil +} + func avifEncoder(img *vips.ImageRef, rawPath string, optimizedPath string) error { var ( buf []byte @@ -139,13 +193,13 @@ func avifEncoder(img *vips.ImageRef, rawPath string, optimizedPath string) error if quality >= 100 { buf, _, err = img.ExportAvif(&vips.AvifExportParams{ Lossless: true, - StripMetadata: true, + StripMetadata: config.Config.StripMetadata, }) } else { buf, _, err = img.ExportAvif(&vips.AvifExportParams{ Quality: quality, Lossless: false, - StripMetadata: true, + StripMetadata: config.Config.StripMetadata, }) } @@ -177,7 +231,7 @@ func webpEncoder(img *vips.ImageRef, rawPath string, optimizedPath string) error // use_lossless_preset = 0; // disable -z option buf, _, err = img.ExportWebp(&vips.WebpExportParams{ Lossless: true, - StripMetadata: true, + StripMetadata: config.Config.StripMetadata, }) } else { // If some special images cannot encode with default ReductionEffort(0), then retry from 0 to 6 @@ -185,7 +239,7 @@ func webpEncoder(img *vips.ImageRef, rawPath string, optimizedPath string) error ep := vips.WebpExportParams{ Quality: quality, Lossless: false, - StripMetadata: true, + StripMetadata: config.Config.StripMetadata, } for i := range 7 { ep.ReductionEffort = i diff --git a/encoder/prefetch.go b/encoder/prefetch.go index 5735abd11..91e22d1c3 100644 --- a/encoder/prefetch.go +++ b/encoder/prefetch.go @@ -33,12 +33,27 @@ func PrefetchImages() { if info.IsDir() { return nil } + if !helper.CheckAllowedType(picAbsPath) { + return nil + } // RawImagePath string, ImgFilename string, reqURI string metadata := helper.ReadMetadata(picAbsPath, "", config.LocalHostAlias) - avif, webp := helper.GenOptimizedAbsPath(metadata, config.LocalHostAlias) - _ = os.MkdirAll(path.Dir(avif), 0755) + avifAbsPath, webpAbsPath, jxlAbsPath := helper.GenOptimizedAbsPath(metadata, config.LocalHostAlias) + + // Using avifAbsPath here is the same as using webpAbsPath/jxlAbsPath + _ = os.MkdirAll(path.Dir(avifAbsPath), 0755) + log.Infof("Prefetching %s", picAbsPath) - go ConvertFilter(picAbsPath, avif, webp, config.ExtraParams{Width: 0, Height: 0}, finishChan) + + // Allow all supported formats + supported := map[string]bool{ + "raw": true, + "webp": true, + "avif": true, + "jxl": true, + } + + go ConvertFilter(picAbsPath, jxlAbsPath, avifAbsPath, webpAbsPath, config.ExtraParams{Width: 0, Height: 0}, supported, finishChan) _ = bar.Add(<-finishChan) return nil }) diff --git a/encoder/process.go b/encoder/process.go index d8af428cc..b149c8901 100644 --- a/encoder/process.go +++ b/encoder/process.go @@ -12,18 +12,69 @@ import ( ) func resizeImage(img *vips.ImageRef, extraParams config.ExtraParams) error { - imgHeightWidthRatio := float32(img.Metadata().Height) / float32(img.Metadata().Width) + imageHeight := img.Height() + imageWidth := img.Width() + + imgHeightWidthRatio := float32(imageHeight) / float32(imageWidth) + + // Here we have width, height and max_width, max_height + // Both pairs cannot be used at the same time + + // max_height and max_width are used to make sure bigger images are resized to max_height and max_width + // e.g, 500x500px image with max_width=200,max_height=100 will be resized to 100x100 + // while smaller images are untouched + + // If both are used, we will use width and height + + if extraParams.MaxHeight > 0 && extraParams.MaxWidth > 0 { + // If any of it exceeds + if imageHeight > extraParams.MaxHeight || imageWidth > extraParams.MaxWidth { + // Check which dimension exceeds most + heightExceedRatio := float32(imageHeight) / float32(extraParams.MaxHeight) + widthExceedRatio := float32(imageWidth) / float32(extraParams.MaxWidth) + // If height exceeds more, like 500x500 -> 200x100 (2.5 < 5) + // Take max_height as new height ,resize and retain ratio + if heightExceedRatio > widthExceedRatio { + err := img.Thumbnail(int(float32(extraParams.MaxHeight)/imgHeightWidthRatio), extraParams.MaxHeight, 0) + if err != nil { + return err + } + } else { + err := img.Thumbnail(extraParams.MaxWidth, int(float32(extraParams.MaxWidth)*imgHeightWidthRatio), 0) + if err != nil { + return err + } + } + } + } + + if extraParams.MaxHeight > 0 && imageHeight > extraParams.MaxHeight && extraParams.MaxWidth == 0 { + err := img.Thumbnail(int(float32(extraParams.MaxHeight)/imgHeightWidthRatio), extraParams.MaxHeight, 0) + if err != nil { + return err + } + } + + if extraParams.MaxWidth > 0 && imageWidth > extraParams.MaxWidth && extraParams.MaxHeight == 0 { + err := img.Thumbnail(extraParams.MaxWidth, int(float32(extraParams.MaxWidth)*imgHeightWidthRatio), 0) + if err != nil { + return err + } + } + if extraParams.Width > 0 && extraParams.Height > 0 { err := img.Thumbnail(extraParams.Width, extraParams.Height, vips.InterestingAttention) if err != nil { return err } - } else if extraParams.Width > 0 && extraParams.Height == 0 { + } + if extraParams.Width > 0 && extraParams.Height == 0 { err := img.Thumbnail(extraParams.Width, int(float32(extraParams.Width)*imgHeightWidthRatio), 0) if err != nil { return err } - } else if extraParams.Height > 0 && extraParams.Width == 0 { + } + if extraParams.Height > 0 && extraParams.Width == 0 { err := img.Thumbnail(int(float32(extraParams.Height)/imgHeightWidthRatio), extraParams.Height, 0) if err != nil { return err @@ -49,6 +100,9 @@ func ResizeItself(raw, dest string, extraParams config.ExtraParams) { return } _ = resizeImage(img, extraParams) + if config.Config.StripMetadata { + img.RemoveMetadata() + } buf, _, _ := img.ExportNative() _ = os.WriteFile(dest, buf, 0600) img.Close() diff --git a/encoder/process_test.go b/encoder/process_test.go new file mode 100644 index 000000000..82ed925f3 --- /dev/null +++ b/encoder/process_test.go @@ -0,0 +1,90 @@ +package encoder + +import ( + "testing" + "webp_server_go/config" + + "github.com/davidbyttow/govips/v2/vips" +) + +func TestResizeImage(t *testing.T) { + img, _ := vips.Black(500, 500) + + // Define the parameters for the test cases + testCases := []struct { + extraParams config.ExtraParams // Extra parameters + expectedH int // Expected height + expectedW int // Expected width + }{ + // Tests for MaxHeight and MaxWidth + // Both extraParams.MaxHeight and extraParams.MaxWidth are 0 + { + extraParams: config.ExtraParams{ + MaxHeight: 0, + MaxWidth: 0, + }, + expectedH: 500, + expectedW: 500, + }, + // Both extraParams.MaxHeight and extraParams.MaxWidth are greater than 0, but the image size is smaller than the limits + { + extraParams: config.ExtraParams{ + MaxHeight: 1000, + MaxWidth: 1000, + }, + expectedH: 500, + expectedW: 500, + }, + // Both extraParams.MaxHeight and extraParams.MaxWidth are greater than 0, and the image exceeds the limits + { + extraParams: config.ExtraParams{ + MaxHeight: 200, + MaxWidth: 200, + }, + expectedH: 200, + expectedW: 200, + }, + // Only MaxHeight is set to 200 + { + extraParams: config.ExtraParams{ + MaxHeight: 200, + MaxWidth: 0, + }, + expectedH: 200, + expectedW: 200, + }, + + // Test for Width and Height + { + extraParams: config.ExtraParams{ + Width: 200, + Height: 200, + }, + expectedH: 200, + expectedW: 200, + }, + { + extraParams: config.ExtraParams{ + Width: 200, + Height: 500, + }, + expectedH: 500, + expectedW: 200, + }, + } + + // Iterate through the test cases and perform the tests + for _, tc := range testCases { + err := resizeImage(img, tc.extraParams) + if err != nil { + t.Errorf("resizeImage failed with error: %v", err) + } + + // Verify if the adjusted image height and width match the expected values + actualH := img.Height() + actualW := img.Width() + if actualH != tc.expectedH || actualW != tc.expectedW { + t.Errorf("resizeImage failed: expected (%d, %d), got (%d, %d)", tc.expectedH, tc.expectedW, actualH, actualW) + } + } +} diff --git a/go.mod b/go.mod index ec12d9440..94bccae12 100644 --- a/go.mod +++ b/go.mod @@ -36,3 +36,5 @@ require ( golang.org/x/text v0.14.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +replace github.com/jeremytorres/rawparser v1.0.2 => github.com/webp-sh/rawparser v0.0.0-20240311121240-15117cd3320a diff --git a/go.sum b/go.sum index 18db44c36..109ca14b2 100644 --- a/go.sum +++ b/go.sum @@ -17,8 +17,6 @@ github.com/h2non/filetype v1.1.4-0.20230123234534-cfcd7d097bc4 h1:k7FGP5I7raiaC3 github.com/h2non/filetype v1.1.4-0.20230123234534-cfcd7d097bc4/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c h1:fEE5/5VNnYUoBOj2I9TP8Jc+a7lge3QWn9DKE7NCwfc= github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c/go.mod h1:ObS/W+h8RYb1Y7fYivughjxojTmIu5iAIjSrSLCLeqE= -github.com/jeremytorres/rawparser v1.0.2 h1:xUHpDBSQv+wZhmi5Dc3zEdlpqQj1X8IPIs8ys78NI/A= -github.com/jeremytorres/rawparser v1.0.2/go.mod h1:X0j2dOqH3ecGRuWvkThgDy+NKAfIwSN9wAOQlMcFOfY= github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= @@ -61,6 +59,8 @@ github.com/valyala/fasthttp v1.52.0 h1:wqBQpxH71XW0e2g+Og4dzQM8pk34aFYlA1Ga8db7g github.com/valyala/fasthttp v1.52.0/go.mod h1:hf5C4QnVMkNXMspnsUlfM3WitlgYflyhHYoKol/szxQ= github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +github.com/webp-sh/rawparser v0.0.0-20240311121240-15117cd3320a h1:yFNUYbDL81wQZ7AQmBhkS+ZDfTugwepVI4LUQ/tQBAc= +github.com/webp-sh/rawparser v0.0.0-20240311121240-15117cd3320a/go.mod h1:X0j2dOqH3ecGRuWvkThgDy+NKAfIwSN9wAOQlMcFOfY= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= diff --git a/handler/router.go b/handler/router.go index 6b2659c41..3328a867d 100644 --- a/handler/router.go +++ b/handler/router.go @@ -4,7 +4,6 @@ import ( "net/http" "net/url" "regexp" - "slices" "strings" "webp_server_go/config" "webp_server_go/encoder" @@ -40,11 +39,15 @@ func Convert(c *fiber.Ctx) error { proxyMode = config.ProxyMode mapMode = false - width, _ = strconv.Atoi(c.Query("width")) // Extra Params - height, _ = strconv.Atoi(c.Query("height")) // Extra Params - extraParams = config.ExtraParams{ - Width: width, - Height: height, + width, _ = strconv.Atoi(c.Query("width")) // Extra Params + height, _ = strconv.Atoi(c.Query("height")) // Extra Params + maxHeight, _ = strconv.Atoi(c.Query("max_height")) // Extra Params + maxWidth, _ = strconv.Atoi(c.Query("max_width")) // Extra Params + extraParams = config.ExtraParams{ + Width: width, + Height: height, + MaxWidth: maxWidth, + MaxHeight: maxHeight, } ) @@ -133,8 +136,11 @@ func Convert(c *fiber.Ctx) error { } supportedFormats := helper.GuessSupportedFormat(reqHeader) - // resize itself and return if only one format(raw) is supported - if len(supportedFormats) == 1 { + // resize itself and return if only raw(original format) is supported + if supportedFormats["raw"] == true && + supportedFormats["webp"] == false && + supportedFormats["avif"] == false && + supportedFormats["jxl"] == false { dest := path.Join(config.Config.ExhaustPath, targetHostName, metadata.Id) if !helper.ImageExists(dest) { encoder.ResizeItself(rawImageAbs, dest, extraParams) @@ -152,16 +158,20 @@ func Convert(c *fiber.Ctx) error { return nil } - avifAbs, webpAbs := helper.GenOptimizedAbsPath(metadata, targetHostName) - encoder.ConvertFilter(rawImageAbs, avifAbs, webpAbs, extraParams, nil) + avifAbs, webpAbs, jxlAbs := helper.GenOptimizedAbsPath(metadata, targetHostName) + // Do the convertion based on supported formats and config + encoder.ConvertFilter(rawImageAbs, jxlAbs, avifAbs, webpAbs, extraParams, supportedFormats, nil) var availableFiles = []string{rawImageAbs} - if slices.Contains(supportedFormats, "avif") { + if supportedFormats["avif"] { availableFiles = append(availableFiles, avifAbs) } - if slices.Contains(supportedFormats, "webp") { + if supportedFormats["webp"] { availableFiles = append(availableFiles, webpAbs) } + if supportedFormats["jxl"] { + availableFiles = append(availableFiles, jxlAbs) + } finalFilename := helper.FindSmallestFiles(availableFiles) contentType := helper.GetFileContentType(finalFilename) diff --git a/handler/router_test.go b/handler/router_test.go index 6e1253d40..5a77eebb1 100644 --- a/handler/router_test.go +++ b/handler/router_test.go @@ -18,12 +18,14 @@ import ( ) var ( - chromeUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36" + chromeUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36" + safariUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1 Safari/605.1.15" + safari17UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.3.1 Safari/605.1.15" // <- Mac with Safari 17 + curlUA = "curl/7.64.1" + acceptWebP = "image/webp,image/apng,image/*,*/*;q=0.8" acceptAvif = "image/avif,image/*,*/*;q=0.8" - acceptLegacy = "image/jpeg" - safariUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1 Safari/605.1.15" - curlUA = "curl/7.64.1" + acceptLegacy = "image/jpeg,image/png" ) func setupParam() { @@ -34,6 +36,7 @@ func setupParam() { config.Metadata = "../metadata" config.RemoteRaw = "../remote-raw" config.ProxyMode = false + config.Config.EnableWebP = true config.Config.EnableAVIF = false config.Config.Quality = 80 config.Config.ImageMap = map[string]string{} @@ -103,7 +106,7 @@ func TestConvertDuplicates(t *testing.T) { // test Chrome for url, respType := range testLink { - for _ = range N { + for range N { resp, data := requestToServer(url, app, chromeUA, acceptWebP) defer resp.Body.Close() contentType := helper.GetContentType(data) diff --git a/helper/helper.go b/helper/helper.go index d832c0573..af709175a 100644 --- a/helper/helper.go +++ b/helper/helper.go @@ -96,12 +96,14 @@ func CheckAllowedType(imgFilename string) bool { return false } -func GenOptimizedAbsPath(metadata config.MetaFile, subdir string) (string, string) { +func GenOptimizedAbsPath(metadata config.MetaFile, subdir string) (string, string, string) { webpFilename := fmt.Sprintf("%s.webp", metadata.Id) avifFilename := fmt.Sprintf("%s.avif", metadata.Id) + jxlFilename := fmt.Sprintf("%s.jxl", metadata.Id) webpAbsolutePath := path.Clean(path.Join(config.Config.ExhaustPath, subdir, webpFilename)) avifAbsolutePath := path.Clean(path.Join(config.Config.ExhaustPath, subdir, avifFilename)) - return avifAbsolutePath, webpAbsolutePath + jxlAbsolutePath := path.Clean(path.Join(config.Config.ExhaustPath, subdir, jxlFilename)) + return avifAbsolutePath, webpAbsolutePath, jxlAbsolutePath } func GetCompressionRate(RawImagePath string, optimizedImg string) string { @@ -119,12 +121,13 @@ func GetCompressionRate(RawImagePath string, optimizedImg string) string { return fmt.Sprintf(`%.2f`, compressionRate) } -func GuessSupportedFormat(header *fasthttp.RequestHeader) []string { +func GuessSupportedFormat(header *fasthttp.RequestHeader) map[string]bool { var ( supported = map[string]bool{ "raw": true, "webp": false, "avif": false, + "jxl": false, } ua = string(header.Peek("user-agent")) @@ -154,14 +157,20 @@ func GuessSupportedFormat(header *fasthttp.RequestHeader) []string { } } - // save true value's key to slice - var accepted []string - for k, v := range supported { - if v { - accepted = append(accepted, k) + // Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15 <- iPad + // Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.3.1 Safari/605.1.15 <- Mac + // Mozilla/5.0 (iPhone; CPU iPhone OS 17_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Mobile/15E148 Safari/604.1 <- iPhone @ Safari + supportedJXLs := []string{"iPhone OS 17", "CPU OS 17", "Version/17"} + if strings.Contains(ua, "iPhone") || strings.Contains(ua, "Macintosh") { + for _, version := range supportedJXLs { + if strings.Contains(ua, version) { + supported["jxl"] = true + break + } } } - return accepted + + return supported } func FindSmallestFiles(files []string) string { diff --git a/helper/helper_test.go b/helper/helper_test.go index 799966bba..645291dca 100644 --- a/helper/helper_test.go +++ b/helper/helper_test.go @@ -1,7 +1,6 @@ package helper import ( - "slices" "testing" "webp_server_go/config" @@ -53,25 +52,41 @@ func TestGuessSupportedFormat(t *testing.T) { name string userAgent string accept string - expected []string + expected map[string]bool }{ + { + name: "WebP/AVIF/JXL Supported", + userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15", // iPad + accept: "image/webp, image/avif", + expected: map[string]bool{ + "raw": true, + "webp": true, + "avif": true, + "jxl": true, + }, + }, { name: "WebP/AVIF Supported", userAgent: "iPhone OS 16", accept: "image/webp, image/png", - expected: []string{"raw", "webp", "avif"}, + expected: map[string]bool{ + "raw": true, + "webp": true, + "avif": true, + "jxl": false, + }, }, { name: "Both Supported", userAgent: "iPhone OS 16", accept: "image/webp, image/avif", - expected: []string{"raw", "webp", "avif"}, + expected: map[string]bool{"raw": true, "webp": true, "avif": true, "jxl": false}, }, { name: "No Supported Formats", userAgent: "Unknown OS", accept: "image/jpeg, image/gif", - expected: []string{"raw"}, + expected: map[string]bool{"raw": true, "webp": false, "avif": false, "jxl": false}, }, } @@ -87,10 +102,8 @@ func TestGuessSupportedFormat(t *testing.T) { t.Errorf("Expected %v, but got %v", test.expected, result) } - for _, format := range test.expected { - if !slices.Contains(result, format) { - t.Errorf("Expected format %s is not in the result", format) - } + for k, v := range test.expected { + assert.Equal(t, v, result[k]) } }) } diff --git a/helper/metadata.go b/helper/metadata.go index 47a3f6d15..ae6068abe 100644 --- a/helper/metadata.go +++ b/helper/metadata.go @@ -18,9 +18,11 @@ func getId(p string) (string, string, string) { parsed, _ := url.Parse(p) width := parsed.Query().Get("width") height := parsed.Query().Get("height") - // santizedPath will be /webp_server.jpg?width=200\u0026height= in local mode when requesting /webp_server.jpg?width=200 + max_width := parsed.Query().Get("max_width") + max_height := parsed.Query().Get("max_height") + // santizedPath will be /webp_server.jpg?width=200\u0026height=\u0026max_width=\u0026max_height= in local mode when requesting /webp_server.jpg?width=200 // santizedPath will be https://docs.webp.sh/images/webp_server.jpg?width=400 in proxy mode when requesting /images/webp_server.jpg?width=400 with IMG_PATH = https://docs.webp.sh - santizedPath := parsed.Path + "?width=" + width + "&height=" + height + santizedPath := parsed.Path + "?width=" + width + "&height=" + height + "&max_width=" + max_width + "&max_height=" + max_height id = HashString(santizedPath) return id, path.Join(config.Config.ImgPath, parsed.Path), santizedPath diff --git a/helper/metadata_test.go b/helper/metadata_test.go index 9f90227ae..1f0015b7b 100644 --- a/helper/metadata_test.go +++ b/helper/metadata_test.go @@ -32,9 +32,9 @@ func TestGetId(t *testing.T) { // Verify the return values parsed, _ := url.Parse(p) - expectedId := HashString(parsed.Path + "?width=400&height=500") + expectedId := HashString(parsed.Path + "?width=400&height=500&max_width=&max_height=") expectedPath := path.Join(config.Config.ImgPath, parsed.Path) - expectedSantizedPath := parsed.Path + "?width=400&height=500" + expectedSantizedPath := parsed.Path + "?width=400&height=500&max_width=&max_height=" if id != expectedId || jointPath != expectedPath || santizedPath != expectedSantizedPath { t.Errorf("Test case 2 failed: Expected (%s, %s, %s), but got (%s, %s, %s)", expectedId, expectedPath, expectedSantizedPath, id, jointPath, santizedPath) diff --git a/webp-server.go b/webp-server.go index 094b72600..785e421a8 100644 --- a/webp-server.go +++ b/webp-server.go @@ -47,7 +47,10 @@ func setupLogger() { TimeFormat: config.TimeDateFormat, })) app.Use(recover.New(recover.Config{})) - log.Infoln("WebP Server Go ready.") + fmt.Println("Allowed file types as source:", config.Config.AllowedTypes) + fmt.Println("Convert to WebP Enabled:", config.Config.EnableWebP) + fmt.Println("Convert to AVIF Enabled:", config.Config.EnableAVIF) + fmt.Println("Convert to JXL Enabled:", config.Config.EnableJXL) } func init() { @@ -92,5 +95,4 @@ func main() { fmt.Println("WebP Server Go is Running on http://" + listenAddress) _ = app.Listen(listenAddress) - }