wood/webp_server_go
1
0
Fork 0
mirror of https://github.com/woodchen-ink/webp_server_go.git synced 2025-07-18 05:32:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

Adds JPEG XL support, max_height/max_width support (#321)

Browse Source 
* WIP JXL

* Fix test

* Tries to fix autobuild

* Tries to fix autobuild

* Add setup go in codeql

* Bump actions version

* Do not print curl output in CI

* Do not print curl output in CI

* Remove Metadata on RAW image

* Update sample config

* better loop

* Prefetch should also respect AllowedType

* Better Export params and UA handle

* Only do conversion on supported formats

* CONVERT_TYPES default to webp only

* CONVERT_TYPES default to webp only

* Add GIF to AllowedTypes

* Update README
This commit is contained in:
Nova Kwok 2024-03-22 15:12:09 +08:00 committed by GitHub
parent ddcb5323b5
commit 43c275e3ec
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
20 changed files with 405 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/CI.yaml vendored
View File

@ -40,7 +40,7 @@ jobs:
submodules: true
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v3
- name: Cache Docker layers
uses: actions/cache@v2

26
.github/workflows/codeql-analysis.yml vendored
View File

@ -15,7 +15,6 @@ on:
push:
branches: [ master ]
pull_request:
# The branches below must be a subset of the branches above
branches: [ master ]
schedule:
- cron: '32 20 * * 2'
@ -33,38 +32,23 @@ jobs:
fail-fast: false
matrix:
language: [ 'go' ]
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
# Learn more about CodeQL language support at https://git.io/codeql-language-support
steps:
- name: Checkout repository
uses: actions/checkout@v3
# Initializes the CodeQL tools for scanning.
- name: Install Go
uses: actions/setup-go@v3
with:
go-version: '1.22'
- name: Initialize CodeQL
uses: github/codeql-action/init@v2
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
# By default, queries listed here will override any specified in a config file.
# Prefix the list here with "+" to use these queries and those in the config file.
# queries: ./path/to/local/query, your-org/your-repo/queries@main
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@v2
# ℹ️ Command-line programs to run using the OS shell.
# 📚 https://git.io/JvXDl
# ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
# and modify them (or add more) to build your code if your project
# uses a compiled language
#- run: |
# make bootstrap
# make release
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v2

4
.github/workflows/integration-test.yaml vendored
View File

@ -32,7 +32,7 @@ jobs:
- name: Send Requests to Server
run: |
cd pics
find * -type f -print | xargs -I {} curl -H "Accept: image/webp" http://localhost:3333/{}
find * -type f -print | xargs -I {} curl -o /dev/null -H "Accept: image/webp" http://localhost:3333/{}
- name: Get container RAM stats
run: |
@ -61,7 +61,7 @@ jobs:
- name: Send Requests to Server
run: |
cd pics
find * -type f -print | xargs -I {} curl -H "Accept: image/webp" http://localhost:3333/{}
find * -type f -print | xargs -I {} curl -o /dev/null -H "Accept: image/webp" http://localhost:3333/{}
- name: Get container RAM stats
run: |

4
.github/workflows/release_docker_image.yaml vendored
View File

@ -43,10 +43,10 @@ jobs:
password: ${{ secrets.GITHUB_TOKEN }}
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
uses: docker/setup-buildx-action@v3
- name: Cache Docker layers
uses: actions/cache@v2

6
README.md
View File

@ -16,7 +16,7 @@ Currently supported image format: JPEG, PNG, BMP, GIF, SVG, HEIC, NEF, WEBP
> e.g When you visit `https://your.website/pics/tsuki.jpg`，it will serve as `image/webp`/`image/avif` format without changing the URL.
>
> GIF image will not be converted to AVIF format even with `ENABLE_AVIF` to `true`, because the converted AVIF image is not animated.
> GIF image will not be converted to AVIF format because the converted AVIF image is not animated.
## Usage with Docker(recommended)
@ -32,8 +32,6 @@ services:
image: webpsh/webp-server-go
# image: ghcr.io/webp-sh/webp_server_go
restart: always
environment:
- MALLOC_ARENA_MAX=1
volumes:
- ./path/to/pics:/opt/pics
- ./exhaust:/opt/exhaust
@ -74,8 +72,6 @@ services:
image: webpsh/webp-server-go
# image: ghcr.io/webp-sh/webp_server_go
restart: always
environment:
- MALLOC_ARENA_MAX=1
volumes:
- ./path/to/pics:/opt/pics
- ./path/to/exhaust:/opt/exhaust

7
config.json
View File

@ -4,12 +4,13 @@
"QUALITY": "80",
"IMG_PATH": "./pics",
"EXHAUST_PATH": "./exhaust",
"ALLOWED_TYPES": ["jpg","png","jpeg","bmp","gif","svg","heic"],
"IMG_MAP": {},
"ENABLE_AVIF": false,
"ALLOWED_TYPES": ["jpg","png","jpeg","gif","bmp","svg","heic","nef"],
"CONVERT_TYPES": ["webp"],
"STRIP_METADATA": true,
"ENABLE_EXTRA_PARAMS": false,
"READ_BUFFER_SIZE": 4096,
"CONCURRENCY": 262144,
"DISABLE_KEEPALIVE": false,
"CACHE_TTL": 259200
}
}

114
config/config.go
View File

@ -6,6 +6,7 @@ import (
"os"
"regexp"
"runtime"
"slices"
"strconv"
"strings"
"time"
@ -28,13 +29,14 @@ const (
"IMG_PATH": "./pics",
"EXHAUST_PATH": "./exhaust",
"IMG_MAP": {},
"ALLOWED_TYPES": ["jpg","png","jpeg","bmp","svg","heic","nef"],
"ENABLE_AVIF": false,
"ENABLE_EXTRA_PARAMS": false
"ALLOWED_TYPES": ["jpg","png","jpeg","gif","bmp","svg","heic","nef"],
"CONVERT_TYPES": ["webp"],
"STRIP_METADATA": true,
"ENABLE_EXTRA_PARAMS": false,
"READ_BUFFER_SIZE": 4096,
"CONCURRENCY": 262144,
"DISABLE_KEEPALIVE": false,
"CACHE_TTL": 259200,
"CACHE_TTL": 259200
}`
)
@ -47,7 +49,7 @@ var (
ProxyMode bool
Prefetch bool
Config = NewWebPConfig()
Version = "0.10.8"
Version = "0.11.0"
WriteLock = cache.New(5*time.Minute, 10*time.Minute)
ConvertLock = cache.New(5*time.Minute, 10*time.Minute)
RemoteRaw = "./remote-raw"
@ -63,32 +65,44 @@ type MetaFile struct {
}
type WebpConfig struct {
Host string `json:"HOST"`
Port string `json:"PORT"`
ImgPath string `json:"IMG_PATH"`
Quality int `json:"QUALITY,string"`
AllowedTypes []string `json:"ALLOWED_TYPES"`
ImageMap map[string]string `json:"IMG_MAP"`
ExhaustPath string `json:"EXHAUST_PATH"`
EnableAVIF bool `json:"ENABLE_AVIF"`
EnableExtraParams bool `json:"ENABLE_EXTRA_PARAMS"`
ReadBufferSize int `json:"READ_BUFFER_SIZE"`
Concurrency int `json:"CONCURRENCY"`
DisableKeepalive bool `json:"DISABLE_KEEPALIVE"`
CacheTTL int `json:"CACHE_TTL"`
Host string `json:"HOST"`
Port string `json:"PORT"`
ImgPath string `json:"IMG_PATH"`
Quality int `json:"QUALITY,string"`
AllowedTypes []string `json:"ALLOWED_TYPES"`
ConvertTypes []string `json:"CONVERT_TYPES"`
ImageMap map[string]string `json:"IMG_MAP"`
ExhaustPath string `json:"EXHAUST_PATH"`
EnableWebP bool `json:"ENABLE_WEBP"`
EnableAVIF bool `json:"ENABLE_AVIF"`
EnableJXL bool `json:"ENABLE_JXL"`
EnableExtraParams bool `json:"ENABLE_EXTRA_PARAMS"`
StripMetadata bool `json:"STRIP_METADATA"`
ReadBufferSize int `json:"READ_BUFFER_SIZE"`
Concurrency int `json:"CONCURRENCY"`
DisableKeepalive bool `json:"DISABLE_KEEPALIVE"`
CacheTTL int `json:"CACHE_TTL"`
}
func NewWebPConfig() *WebpConfig {
return &WebpConfig{
Host: "0.0.0.0",
Port: "3333",
ImgPath: "./pics",
Quality: 80,
AllowedTypes: []string{"jpg", "png", "jpeg", "bmp", "svg", "nef", "heic", "webp"},
ImageMap: map[string]string{},
ExhaustPath: "./exhaust",
EnableAVIF: false,
Host: "0.0.0.0",
Port: "3333",
ImgPath: "./pics",
Quality: 80,
AllowedTypes: []string{"jpg", "png", "jpeg", "bmp", "gif", "svg", "nef", "heic", "webp"},
ConvertTypes: []string{"webp"},
ImageMap: map[string]string{},
ExhaustPath: "./exhaust",
EnableWebP: false,
EnableAVIF: false,
EnableJXL: false,
EnableExtraParams: false,
StripMetadata: true,
ReadBufferSize: 4096,
Concurrency: 262144,
DisableKeepalive: false,
@ -115,6 +129,16 @@ func LoadConfig() {
switchProxyMode()
Config.ImageMap = parseImgMap(Config.ImageMap)
if slices.Contains(Config.ConvertTypes, "webp") {
Config.EnableWebP = true
}
if slices.Contains(Config.ConvertTypes, "avif") {
Config.EnableAVIF = true
}
if slices.Contains(Config.ConvertTypes, "jxl") {
Config.EnableJXL = true
}
// Read from ENV for override
if os.Getenv("WEBP_HOST") != "" {
Config.Host = os.Getenv("WEBP_HOST")
@ -139,16 +163,24 @@ func LoadConfig() {
if os.Getenv("WEBP_ALLOWED_TYPES") != "" {
Config.AllowedTypes = strings.Split(os.Getenv("WEBP_ALLOWED_TYPES"), ",")
}
if os.Getenv("WEBP_ENABLE_AVIF") != "" {
enableAVIF := os.Getenv("WEBP_ENABLE_AVIF")
if enableAVIF == "true" {
// Override enabled convert types
if os.Getenv("WEBP_CONVERT_TYPES") != "" {
Config.ConvertTypes = strings.Split(os.Getenv("WEBP_CONVERT_TYPES"), ",")
Config.EnableWebP = false
Config.EnableAVIF = false
Config.EnableJXL = false
if slices.Contains(Config.ConvertTypes, "webp") {
Config.EnableWebP = true
}
if slices.Contains(Config.ConvertTypes, "avif") {
Config.EnableAVIF = true
} else if enableAVIF == "false" {
Config.EnableAVIF = false
} else {
log.Warnf("WEBP_ENABLE_AVIF is not a valid boolean, using value in config.json %t", Config.EnableAVIF)
}
if slices.Contains(Config.ConvertTypes, "jxl") {
Config.EnableJXL = true
}
}
if os.Getenv("WEBP_ENABLE_EXTRA_PARAMS") != "" {
enableExtraParams := os.Getenv("WEBP_ENABLE_EXTRA_PARAMS")
if enableExtraParams == "true" {
@ -159,6 +191,16 @@ func LoadConfig() {
log.Warnf("WEBP_ENABLE_EXTRA_PARAMS is not a valid boolean, using value in config.json %t", Config.EnableExtraParams)
}
}
if os.Getenv("WEBP_STRIP_METADATA") != "" {
stripMetadata := os.Getenv("WEBP_STRIP_METADATA")
if stripMetadata == "true" {
Config.StripMetadata = true
} else if stripMetadata == "false" {
Config.StripMetadata = false
} else {
log.Warnf("WEBP_STRIP_METADATA is not a valid boolean, using value in config.json %t", Config.StripMetadata)
}
}
if os.Getenv("WEBP_IMG_MAP") != "" {
// TODO
}
@ -223,8 +265,10 @@ func parseImgMap(imgMap map[string]string) map[string]string {
}
type ExtraParams struct {
Width int // in px
Height int // in px
Width int // in px
Height int // in px
MaxWidth int // in px
MaxHeight int // in px
}
func switchProxyMode() {

70
encoder/encoder.go
View File

@ -33,7 +33,7 @@ func init() {
intMinusOne.Set(-1)
}
func ConvertFilter(rawPath, avifPath, webpPath string, extraParams config.ExtraParams, c chan int) {
func ConvertFilter(rawPath, jxlPath, avifPath, webpPath string, extraParams config.ExtraParams, supportedFormats map[string]bool, c chan int) {
// Wait for the conversion to complete and return the converted image
retryDelay := 100 * time.Millisecond // Initial retry delay
@ -53,8 +53,8 @@ func ConvertFilter(rawPath, avifPath, webpPath string, extraParams config.ExtraP
defer config.ConvertLock.Delete(rawPath)
var wg sync.WaitGroup
wg.Add(2)
if !helper.ImageExists(avifPath) && config.Config.EnableAVIF {
wg.Add(3)
if !helper.ImageExists(avifPath) && config.Config.EnableAVIF && supportedFormats["avif"] {
go func() {
err := convertImage(rawPath, avifPath, "avif", extraParams)
if err != nil {
@ -66,7 +66,7 @@ func ConvertFilter(rawPath, avifPath, webpPath string, extraParams config.ExtraP
wg.Done()
}
if !helper.ImageExists(webpPath) {
if !helper.ImageExists(webpPath) && config.Config.EnableWebP && supportedFormats["webp"] {
go func() {
err := convertImage(rawPath, webpPath, "webp", extraParams)
if err != nil {
@ -77,6 +77,19 @@ func ConvertFilter(rawPath, avifPath, webpPath string, extraParams config.ExtraP
} else {
wg.Done()
}
if !helper.ImageExists(jxlPath) && config.Config.EnableJXL && supportedFormats["jxl"] {
go func() {
err := convertImage(rawPath, jxlPath, "jxl", extraParams)
if err != nil {
log.Errorln(err)
}
defer wg.Done()
}()
} else {
wg.Done()
}
wg.Wait()
if c != nil {
@ -123,11 +136,52 @@ func convertImage(rawPath, optimizedPath, imageType string, extraParams config.E
err = webpEncoder(img, rawPath, optimizedPath)
case "avif":
err = avifEncoder(img, rawPath, optimizedPath)
case "jxl":
err = jxlEncoder(img, rawPath, optimizedPath)
}
return err
}
func jxlEncoder(img *vips.ImageRef, rawPath string, optimizedPath string) error {
var (
buf []byte
quality = config.Config.Quality
err error
)
// If quality >= 100, we use lossless mode
if quality >= 100 {
buf, _, err = img.ExportJxl(&vips.JxlExportParams{
Effort: 1,
Tier: 4,
Lossless: true,
Distance: 1.0,
})
} else {
buf, _, err = img.ExportJxl(&vips.JxlExportParams{
Effort: 1,
Tier: 4,
Quality: quality,
Lossless: false,
Distance: 1.0,
})
}
if err != nil {
log.Warnf("Can't encode source image: %v to JXL", err)
return err
}
if err := os.WriteFile(optimizedPath, buf, 0600); err != nil {
log.Error(err)
return err
}
convertLog("JXL", rawPath, optimizedPath, quality)
return nil
}
func avifEncoder(img *vips.ImageRef, rawPath string, optimizedPath string) error {
var (
buf []byte
@ -139,13 +193,13 @@ func avifEncoder(img *vips.ImageRef, rawPath string, optimizedPath string) error
if quality >= 100 {
buf, _, err = img.ExportAvif(&vips.AvifExportParams{
Lossless: true,
StripMetadata: true,
StripMetadata: config.Config.StripMetadata,
})
} else {
buf, _, err = img.ExportAvif(&vips.AvifExportParams{
Quality: quality,
Lossless: false,
StripMetadata: true,
StripMetadata: config.Config.StripMetadata,
})
}
@ -177,7 +231,7 @@ func webpEncoder(img *vips.ImageRef, rawPath string, optimizedPath string) error
// use_lossless_preset = 0; // disable -z option
buf, _, err = img.ExportWebp(&vips.WebpExportParams{
Lossless: true,
StripMetadata: true,
StripMetadata: config.Config.StripMetadata,
})
} else {
// If some special images cannot encode with default ReductionEffort(0), then retry from 0 to 6
@ -185,7 +239,7 @@ func webpEncoder(img *vips.ImageRef, rawPath string, optimizedPath string) error
ep := vips.WebpExportParams{
Quality: quality,
Lossless: false,
StripMetadata: true,
StripMetadata: config.Config.StripMetadata,
}
for i := range 7 {
ep.ReductionEffort = i

21
encoder/prefetch.go
View File

@ -33,12 +33,27 @@ func PrefetchImages() {
if info.IsDir() {
return nil
}
if !helper.CheckAllowedType(picAbsPath) {
return nil
}
// RawImagePath string, ImgFilename string, reqURI string
metadata := helper.ReadMetadata(picAbsPath, "", config.LocalHostAlias)
avif, webp := helper.GenOptimizedAbsPath(metadata, config.LocalHostAlias)
_ = os.MkdirAll(path.Dir(avif), 0755)
avifAbsPath, webpAbsPath, jxlAbsPath := helper.GenOptimizedAbsPath(metadata, config.LocalHostAlias)
// Using avifAbsPath here is the same as using webpAbsPath/jxlAbsPath
_ = os.MkdirAll(path.Dir(avifAbsPath), 0755)
log.Infof("Prefetching %s", picAbsPath)
go ConvertFilter(picAbsPath, avif, webp, config.ExtraParams{Width: 0, Height: 0}, finishChan)
// Allow all supported formats
supported := map[string]bool{
"raw": true,
"webp": true,
"avif": true,
"jxl": true,
}
go ConvertFilter(picAbsPath, jxlAbsPath, avifAbsPath, webpAbsPath, config.ExtraParams{Width: 0, Height: 0}, supported, finishChan)
_ = bar.Add(<-finishChan)
return nil
})

60
encoder/process.go
View File

@ -12,18 +12,69 @@ import (
)
func resizeImage(img *vips.ImageRef, extraParams config.ExtraParams) error {
imgHeightWidthRatio := float32(img.Metadata().Height) / float32(img.Metadata().Width)
imageHeight := img.Height()
imageWidth := img.Width()
imgHeightWidthRatio := float32(imageHeight) / float32(imageWidth)
// Here we have width, height and max_width, max_height
// Both pairs cannot be used at the same time
// max_height and max_width are used to make sure bigger images are resized to max_height and max_width
// e.g, 500x500px image with max_width=200,max_height=100 will be resized to 100x100
// while smaller images are untouched
// If both are used, we will use width and height
if extraParams.MaxHeight > 0 && extraParams.MaxWidth > 0 {
// If any of it exceeds
if imageHeight > extraParams.MaxHeight || imageWidth > extraParams.MaxWidth {
// Check which dimension exceeds most
heightExceedRatio := float32(imageHeight) / float32(extraParams.MaxHeight)
widthExceedRatio := float32(imageWidth) / float32(extraParams.MaxWidth)
// If height exceeds more, like 500x500 -> 200x100 (2.5 < 5)
// Take max_height as new height ,resize and retain ratio
if heightExceedRatio > widthExceedRatio {
err := img.Thumbnail(int(float32(extraParams.MaxHeight)/imgHeightWidthRatio), extraParams.MaxHeight, 0)
if err != nil {
return err
}
} else {
err := img.Thumbnail(extraParams.MaxWidth, int(float32(extraParams.MaxWidth)*imgHeightWidthRatio), 0)
if err != nil {
return err
}
}
}
}
if extraParams.MaxHeight > 0 && imageHeight > extraParams.MaxHeight && extraParams.MaxWidth == 0 {
err := img.Thumbnail(int(float32(extraParams.MaxHeight)/imgHeightWidthRatio), extraParams.MaxHeight, 0)
if err != nil {
return err
}
}
if extraParams.MaxWidth > 0 && imageWidth > extraParams.MaxWidth && extraParams.MaxHeight == 0 {
err := img.Thumbnail(extraParams.MaxWidth, int(float32(extraParams.MaxWidth)*imgHeightWidthRatio), 0)
if err != nil {
return err
}
}
if extraParams.Width > 0 && extraParams.Height > 0 {
err := img.Thumbnail(extraParams.Width, extraParams.Height, vips.InterestingAttention)
if err != nil {
return err
}
} else if extraParams.Width > 0 && extraParams.Height == 0 {
}
if extraParams.Width > 0 && extraParams.Height == 0 {
err := img.Thumbnail(extraParams.Width, int(float32(extraParams.Width)*imgHeightWidthRatio), 0)
if err != nil {
return err
}
} else if extraParams.Height > 0 && extraParams.Width == 0 {
}
if extraParams.Height > 0 && extraParams.Width == 0 {
err := img.Thumbnail(int(float32(extraParams.Height)/imgHeightWidthRatio), extraParams.Height, 0)
if err != nil {
return err
@ -49,6 +100,9 @@ func ResizeItself(raw, dest string, extraParams config.ExtraParams) {
return
}
_ = resizeImage(img, extraParams)
if config.Config.StripMetadata {
img.RemoveMetadata()
}
buf, _, _ := img.ExportNative()
_ = os.WriteFile(dest, buf, 0600)
img.Close()

90
encoder/process_test.go Normal file
View File

@ -0,0 +1,90 @@
package encoder
import (
"testing"
"webp_server_go/config"
"github.com/davidbyttow/govips/v2/vips"
)
func TestResizeImage(t *testing.T) {
img, _ := vips.Black(500, 500)
// Define the parameters for the test cases
testCases := []struct {
extraParams config.ExtraParams // Extra parameters
expectedH int // Expected height
expectedW int // Expected width
}{
// Tests for MaxHeight and MaxWidth
// Both extraParams.MaxHeight and extraParams.MaxWidth are 0
{
extraParams: config.ExtraParams{
MaxHeight: 0,
MaxWidth: 0,
},
expectedH: 500,
expectedW: 500,
},
// Both extraParams.MaxHeight and extraParams.MaxWidth are greater than 0, but the image size is smaller than the limits
{
extraParams: config.ExtraParams{
MaxHeight: 1000,
MaxWidth: 1000,
},
expectedH: 500,
expectedW: 500,
},
// Both extraParams.MaxHeight and extraParams.MaxWidth are greater than 0, and the image exceeds the limits
{
extraParams: config.ExtraParams{
MaxHeight: 200,
MaxWidth: 200,
},
expectedH: 200,
expectedW: 200,
},
// Only MaxHeight is set to 200
{
extraParams: config.ExtraParams{
MaxHeight: 200,
MaxWidth: 0,
},
expectedH: 200,
expectedW: 200,
},
// Test for Width and Height
{
extraParams: config.ExtraParams{
Width: 200,
Height: 200,
},
expectedH: 200,
expectedW: 200,
},
{
extraParams: config.ExtraParams{
Width: 200,
Height: 500,
},
expectedH: 500,
expectedW: 200,
},
}
// Iterate through the test cases and perform the tests
for _, tc := range testCases {
err := resizeImage(img, tc.extraParams)
if err != nil {
t.Errorf("resizeImage failed with error: %v", err)
}
// Verify if the adjusted image height and width match the expected values
actualH := img.Height()
actualW := img.Width()
if actualH != tc.expectedH || actualW != tc.expectedW {
t.Errorf("resizeImage failed: expected (%d, %d), got (%d, %d)", tc.expectedH, tc.expectedW, actualH, actualW)
}
}
}

2
go.mod
View File

@ -36,3 +36,5 @@ require (
golang.org/x/text v0.14.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
replace github.com/jeremytorres/rawparser v1.0.2 => github.com/webp-sh/rawparser v0.0.0-20240311121240-15117cd3320a

4
go.sum
View File

@ -17,8 +17,6 @@ github.com/h2non/filetype v1.1.4-0.20230123234534-cfcd7d097bc4 h1:k7FGP5I7raiaC3
github.com/h2non/filetype v1.1.4-0.20230123234534-cfcd7d097bc4/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c h1:fEE5/5VNnYUoBOj2I9TP8Jc+a7lge3QWn9DKE7NCwfc=
github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c/go.mod h1:ObS/W+h8RYb1Y7fYivughjxojTmIu5iAIjSrSLCLeqE=
github.com/jeremytorres/rawparser v1.0.2 h1:xUHpDBSQv+wZhmi5Dc3zEdlpqQj1X8IPIs8ys78NI/A=
github.com/jeremytorres/rawparser v1.0.2/go.mod h1:X0j2dOqH3ecGRuWvkThgDy+NKAfIwSN9wAOQlMcFOfY=
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI=
github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
@ -61,6 +59,8 @@ github.com/valyala/fasthttp v1.52.0 h1:wqBQpxH71XW0e2g+Og4dzQM8pk34aFYlA1Ga8db7g
github.com/valyala/fasthttp v1.52.0/go.mod h1:hf5C4QnVMkNXMspnsUlfM3WitlgYflyhHYoKol/szxQ=
github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
github.com/webp-sh/rawparser v0.0.0-20240311121240-15117cd3320a h1:yFNUYbDL81wQZ7AQmBhkS+ZDfTugwepVI4LUQ/tQBAc=
github.com/webp-sh/rawparser v0.0.0-20240311121240-15117cd3320a/go.mod h1:X0j2dOqH3ecGRuWvkThgDy+NKAfIwSN9wAOQlMcFOfY=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=

34
handler/router.go
View File

@ -4,7 +4,6 @@ import (
"net/http"
"net/url"
"regexp"
"slices"
"strings"
"webp_server_go/config"
"webp_server_go/encoder"
@ -40,11 +39,15 @@ func Convert(c *fiber.Ctx) error {
proxyMode = config.ProxyMode
mapMode = false
width, _ = strconv.Atoi(c.Query("width")) // Extra Params
height, _ = strconv.Atoi(c.Query("height")) // Extra Params
extraParams = config.ExtraParams{
Width: width,
Height: height,
width, _ = strconv.Atoi(c.Query("width")) // Extra Params
height, _ = strconv.Atoi(c.Query("height")) // Extra Params
maxHeight, _ = strconv.Atoi(c.Query("max_height")) // Extra Params
maxWidth, _ = strconv.Atoi(c.Query("max_width")) // Extra Params
extraParams = config.ExtraParams{
Width: width,
Height: height,
MaxWidth: maxWidth,
MaxHeight: maxHeight,
}
)
@ -133,8 +136,11 @@ func Convert(c *fiber.Ctx) error {
}
supportedFormats := helper.GuessSupportedFormat(reqHeader)
// resize itself and return if only one format(raw) is supported
if len(supportedFormats) == 1 {
// resize itself and return if only raw(original format) is supported
if supportedFormats["raw"] == true &&
supportedFormats["webp"] == false &&
supportedFormats["avif"] == false &&
supportedFormats["jxl"] == false {
dest := path.Join(config.Config.ExhaustPath, targetHostName, metadata.Id)
if !helper.ImageExists(dest) {
encoder.ResizeItself(rawImageAbs, dest, extraParams)
@ -152,16 +158,20 @@ func Convert(c *fiber.Ctx) error {
return nil
}
avifAbs, webpAbs := helper.GenOptimizedAbsPath(metadata, targetHostName)
encoder.ConvertFilter(rawImageAbs, avifAbs, webpAbs, extraParams, nil)
avifAbs, webpAbs, jxlAbs := helper.GenOptimizedAbsPath(metadata, targetHostName)
// Do the convertion based on supported formats and config
encoder.ConvertFilter(rawImageAbs, jxlAbs, avifAbs, webpAbs, extraParams, supportedFormats, nil)
var availableFiles = []string{rawImageAbs}
if slices.Contains(supportedFormats, "avif") {
if supportedFormats["avif"] {
availableFiles = append(availableFiles, avifAbs)
}
if slices.Contains(supportedFormats, "webp") {
if supportedFormats["webp"] {
availableFiles = append(availableFiles, webpAbs)
}
if supportedFormats["jxl"] {
availableFiles = append(availableFiles, jxlAbs)
}
finalFilename := helper.FindSmallestFiles(availableFiles)
contentType := helper.GetFileContentType(finalFilename)

13
handler/router_test.go
View File

@ -18,12 +18,14 @@ import (
)
var (
chromeUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36"
chromeUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36"
safariUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1 Safari/605.1.15"
safari17UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.3.1 Safari/605.1.15" // <- Mac with Safari 17
curlUA = "curl/7.64.1"
acceptWebP = "image/webp,image/apng,image/*,*/*;q=0.8"
acceptAvif = "image/avif,image/*,*/*;q=0.8"
acceptLegacy = "image/jpeg"
safariUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1 Safari/605.1.15"
curlUA = "curl/7.64.1"
acceptLegacy = "image/jpeg,image/png"
)
func setupParam() {
@ -34,6 +36,7 @@ func setupParam() {
config.Metadata = "../metadata"
config.RemoteRaw = "../remote-raw"
config.ProxyMode = false
config.Config.EnableWebP = true
config.Config.EnableAVIF = false
config.Config.Quality = 80
config.Config.ImageMap = map[string]string{}
@ -103,7 +106,7 @@ func TestConvertDuplicates(t *testing.T) {
// test Chrome
for url, respType := range testLink {
for _ = range N {
for range N {
resp, data := requestToServer(url, app, chromeUA, acceptWebP)
defer resp.Body.Close()
contentType := helper.GetContentType(data)

27
helper/helper.go
View File

@ -96,12 +96,14 @@ func CheckAllowedType(imgFilename string) bool {
return false
}
func GenOptimizedAbsPath(metadata config.MetaFile, subdir string) (string, string) {
func GenOptimizedAbsPath(metadata config.MetaFile, subdir string) (string, string, string) {
webpFilename := fmt.Sprintf("%s.webp", metadata.Id)
avifFilename := fmt.Sprintf("%s.avif", metadata.Id)
jxlFilename := fmt.Sprintf("%s.jxl", metadata.Id)
webpAbsolutePath := path.Clean(path.Join(config.Config.ExhaustPath, subdir, webpFilename))
avifAbsolutePath := path.Clean(path.Join(config.Config.ExhaustPath, subdir, avifFilename))
return avifAbsolutePath, webpAbsolutePath
jxlAbsolutePath := path.Clean(path.Join(config.Config.ExhaustPath, subdir, jxlFilename))
return avifAbsolutePath, webpAbsolutePath, jxlAbsolutePath
}
func GetCompressionRate(RawImagePath string, optimizedImg string) string {
@ -119,12 +121,13 @@ func GetCompressionRate(RawImagePath string, optimizedImg string) string {
return fmt.Sprintf(`%.2f`, compressionRate)
}
func GuessSupportedFormat(header *fasthttp.RequestHeader) []string {
func GuessSupportedFormat(header *fasthttp.RequestHeader) map[string]bool {
var (
supported = map[string]bool{
"raw": true,
"webp": false,
"avif": false,
"jxl": false,
}
ua = string(header.Peek("user-agent"))
@ -154,14 +157,20 @@ func GuessSupportedFormat(header *fasthttp.RequestHeader) []string {
}
}
// save true value's key to slice
var accepted []string
for k, v := range supported {
if v {
accepted = append(accepted, k)
// Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15 <- iPad
// Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.3.1 Safari/605.1.15 <- Mac
// Mozilla/5.0 (iPhone; CPU iPhone OS 17_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Mobile/15E148 Safari/604.1 <- iPhone @ Safari
supportedJXLs := []string{"iPhone OS 17", "CPU OS 17", "Version/17"}
if strings.Contains(ua, "iPhone") || strings.Contains(ua, "Macintosh") {
for _, version := range supportedJXLs {
if strings.Contains(ua, version) {
supported["jxl"] = true
break
}
}
}
return accepted
return supported
}
func FindSmallestFiles(files []string) string {

31
helper/helper_test.go
View File

@ -1,7 +1,6 @@
package helper
import (
"slices"
"testing"
"webp_server_go/config"
@ -53,25 +52,41 @@ func TestGuessSupportedFormat(t *testing.T) {
name string
userAgent string
accept string
expected []string
expected map[string]bool
}{
{
name: "WebP/AVIF/JXL Supported",
userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15", // iPad
accept: "image/webp, image/avif",
expected: map[string]bool{
"raw": true,
"webp": true,
"avif": true,
"jxl": true,
},
},
{
name: "WebP/AVIF Supported",
userAgent: "iPhone OS 16",
accept: "image/webp, image/png",
expected: []string{"raw", "webp", "avif"},
expected: map[string]bool{
"raw": true,
"webp": true,
"avif": true,
"jxl": false,
},
},
{
name: "Both Supported",
userAgent: "iPhone OS 16",
accept: "image/webp, image/avif",
expected: []string{"raw", "webp", "avif"},
expected: map[string]bool{"raw": true, "webp": true, "avif": true, "jxl": false},
},
{
name: "No Supported Formats",
userAgent: "Unknown OS",
accept: "image/jpeg, image/gif",
expected: []string{"raw"},
expected: map[string]bool{"raw": true, "webp": false, "avif": false, "jxl": false},
},
}
@ -87,10 +102,8 @@ func TestGuessSupportedFormat(t *testing.T) {
t.Errorf("Expected %v, but got %v", test.expected, result)
}
for _, format := range test.expected {
if !slices.Contains(result, format) {
t.Errorf("Expected format %s is not in the result", format)
}
for k, v := range test.expected {
assert.Equal(t, v, result[k])
}
})
}

6
helper/metadata.go
View File

@ -18,9 +18,11 @@ func getId(p string) (string, string, string) {
parsed, _ := url.Parse(p)
width := parsed.Query().Get("width")
height := parsed.Query().Get("height")
// santizedPath will be /webp_server.jpg?width=200\u0026height= in local mode when requesting /webp_server.jpg?width=200
max_width := parsed.Query().Get("max_width")
max_height := parsed.Query().Get("max_height")
// santizedPath will be /webp_server.jpg?width=200\u0026height=\u0026max_width=\u0026max_height= in local mode when requesting /webp_server.jpg?width=200
// santizedPath will be https://docs.webp.sh/images/webp_server.jpg?width=400 in proxy mode when requesting /images/webp_server.jpg?width=400 with IMG_PATH = https://docs.webp.sh
santizedPath := parsed.Path + "?width=" + width + "&height=" + height
santizedPath := parsed.Path + "?width=" + width + "&height=" + height + "&max_width=" + max_width + "&max_height=" + max_height
id = HashString(santizedPath)
return id, path.Join(config.Config.ImgPath, parsed.Path), santizedPath

4
helper/metadata_test.go
View File

@ -32,9 +32,9 @@ func TestGetId(t *testing.T) {
// Verify the return values
parsed, _ := url.Parse(p)
expectedId := HashString(parsed.Path + "?width=400&height=500")
expectedId := HashString(parsed.Path + "?width=400&height=500&max_width=&max_height=")
expectedPath := path.Join(config.Config.ImgPath, parsed.Path)
expectedSantizedPath := parsed.Path + "?width=400&height=500"
expectedSantizedPath := parsed.Path + "?width=400&height=500&max_width=&max_height="
if id != expectedId || jointPath != expectedPath || santizedPath != expectedSantizedPath {
t.Errorf("Test case 2 failed: Expected (%s, %s, %s), but got (%s, %s, %s)",
expectedId, expectedPath, expectedSantizedPath, id, jointPath, santizedPath)

6
webp-server.go
View File

@ -47,7 +47,10 @@ func setupLogger() {
TimeFormat: config.TimeDateFormat,
}))
app.Use(recover.New(recover.Config{}))
log.Infoln("WebP Server Go ready.")
fmt.Println("Allowed file types as source:", config.Config.AllowedTypes)
fmt.Println("Convert to WebP Enabled:", config.Config.EnableWebP)
fmt.Println("Convert to AVIF Enabled:", config.Config.EnableAVIF)
fmt.Println("Convert to JXL Enabled:", config.Config.EnableJXL)
}
func init() {
@ -92,5 +95,4 @@ func main() {
fmt.Println("WebP Server Go is Running on http://" + listenAddress)
_ = app.Listen(listenAddress)
}