wood/webp_server_go
1
0
Fork 0
mirror of https://github.com/woodchen-ink/webp_server_go.git synced 2025-07-18 13:42:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactor review (#220)

Browse Source 
* runnable

* convert is working

* some refactoring

* update go.mod

* fix some TODOs

* add TODO

* update go mod

* rebase onto master

* fix #234

2: 5.9s - 7.6MB
4: 26s - 6.9MB

* fix malloc tests

* fix malloc tests

* remote TODO

* add X-Real-IP #236

* Better localRawImagePath

* remove some wrong comments

* Bump version to 0.9.0

---------

Co-authored-by: n0vad3v <n0vad3v@riseup.net>
This commit is contained in:
Benny 2023-06-27 15:43:43 +02:00 committed by GitHub
parent a8090ff47e
commit 23bbed8ce6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 705 additions and 1387 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.github/workflows/integration-test.yaml vendored
View File

@ -26,7 +26,7 @@ jobs:
- name: Start the container
run: |
cp tests/glib_malloc/docker-compose.yml ./
cp malloc_tests/glib_malloc/docker-compose.yml ./
docker-compose up -d
- name: Send Requests to Server
@ -55,7 +55,7 @@ jobs:
- name: Start the container
run: |
cp tests/jemalloc/docker-compose.yml ./
cp malloc_tests/jemalloc/docker-compose.yml ./
docker-compose up -d
- name: Send Requests to Server
@ -65,4 +65,4 @@ jobs:
- name: Get container RAM stats
run: |
docker stats --no-stream
docker stats --no-stream

2
.gitignore vendored
View File

@ -23,3 +23,5 @@ builds/
/.idea/webp_server_go.iml
remote-raw/
coverage.txt
.DS_Store
/webp_server_go

6
Makefile
View File

@ -35,8 +35,8 @@ test: static-check
go test -v -coverprofile=coverage.txt -covermode=atomic
clean:
rm -rf builds
rm -rf prefetch
rm -rf builds prefetch remote-raw exhaust tools coverage.txt
docker:
DOCKER_BUILDKIT=1 docker build -t webpsh/webps .
DOCKER_BUILDKIT=1 docker build -t webpsh/webps .

71
config.go
View File

@ -1,71 +0,0 @@
package main
import "fmt"
type Config struct {
Host string `json:"HOST"`
Port string `json:"PORT"`
ImgPath string `json:"IMG_PATH"`
Quality int `json:"QUALITY,string"`
AllowedTypes []string `json:"ALLOWED_TYPES"`
ExhaustPath string `json:"EXHAUST_PATH"`
EnableAVIF bool `json:"ENABLE_AVIF"`
EnableExtraParams bool `json:"ENABLE_EXTRA_PARAMS"`
}
type ExtraParams struct {
Width int // in px
Height int // in px
}
// String : convert ExtraParams to string, used to generate cache path
func (e *ExtraParams) String() string {
return fmt.Sprintf("_width=%d&height=%d", e.Width, e.Height)
}
var (
configPath string
jobs int
dumpConfig, dumpSystemd bool
verboseMode, showVersion bool
prefetch, proxyMode bool
remoteRaw = "remote-raw"
config Config
version = "0.9.0"
)
const (
sampleConfig = `
{
"HOST": "127.0.0.1",
"PORT": "3333",
"QUALITY": "80",
"IMG_PATH": "./pics",
"EXHAUST_PATH": "./exhaust",
"ALLOWED_TYPES": ["jpg","png","jpeg","bmp","gif"],
"ENABLE_AVIF": false,
"ENABLE_EXTRA_PARAMS": false
}`
sampleSystemd = `
[Unit]
Description=WebP Server Go
Documentation=https://github.com/webp-sh/webp_server_go
After=nginx.target
[Service]
Type=simple
StandardError=journal
WorkingDirectory=/opt/webps
ExecStart=/opt/webps/webp-server --config /opt/webps/config.json
Restart=always
RestartSec=3s
[Install]
WantedBy=multi-user.target`
)
const (
webpMax = 16383
avifMax = 65536
)

115
config/config.go Normal file
View File

@ -0,0 +1,115 @@
package config
import (
"encoding/json"
"flag"
"fmt"
"os"
"regexp"
"runtime"
"time"
"github.com/patrickmn/go-cache"
log "github.com/sirupsen/logrus"
)
const (
TimeDateFormat = "2006-01-02 15:04:05"
FiberLogFormat = "${ip} - [${time}] ${method} ${url} ${status} ${referer} ${ua}\n"
WebpMax = 16383
AvifMax = 65536
RemoteRaw = "remote-raw"
SampleConfig = `
{
"HOST": "127.0.0.1",
"PORT": "3333",
"QUALITY": "80",
"IMG_PATH": "./pics",
"EXHAUST_PATH": "./exhaust",
"ALLOWED_TYPES": ["jpg","png","jpeg","bmp"],
"ENABLE_AVIF": false,
"ENABLE_EXTRA_PARAMS": false
}`
SampleSystemd = `
[Unit]
Description=WebP Server Go
Documentation=https://github.com/webp-sh/webp_server_go
After=nginx.target
[Service]
Type=simple
StandardError=journal
WorkingDirectory=/opt/webps
ExecStart=/opt/webps/webp-server --config /opt/webps/config.json
Restart=always
RestartSec=3s
[Install]
WantedBy=multi-user.target`
)
var (
configPath string
Jobs int
DumpSystemd bool
DumpConfig bool
ShowVersion bool
ProxyMode bool
Prefetch bool
Config jsonFile
Version = "0.9.0"
WriteLock = cache.New(5*time.Minute, 10*time.Minute)
)
type jsonFile struct {
Host string `json:"HOST"`
Port string `json:"PORT"`
ImgPath string `json:"IMG_PATH"`
Quality int `json:"QUALITY,string"`
AllowedTypes []string `json:"ALLOWED_TYPES"`
ExhaustPath string `json:"EXHAUST_PATH"`
EnableAVIF bool `json:"ENABLE_AVIF"`
EnableExtraParams bool `json:"ENABLE_EXTRA_PARAMS"`
}
func init() {
flag.StringVar(&configPath, "config", "config.json", "/path/to/config.json. (Default: ./config.json)")
flag.BoolVar(&Prefetch, "prefetch", false, "Prefetch and convert image to webp")
flag.IntVar(&Jobs, "jobs", runtime.NumCPU(), "Prefetch thread, default is all.")
flag.BoolVar(&DumpConfig, "dump-config", false, "Print sample config.json")
flag.BoolVar(&DumpSystemd, "dump-systemd", false, "Print sample systemd service file.")
flag.BoolVar(&ShowVersion, "V", false, "Show version information.")
flag.Parse()
Config = loadConfig()
switchProxyMode()
}
func loadConfig() (config jsonFile) {
jsonObject, err := os.Open(configPath)
if err != nil {
log.Fatal(err)
}
decoder := json.NewDecoder(jsonObject)
_ = decoder.Decode(&config)
_ = jsonObject.Close()
return config
}
type ExtraParams struct {
Width int // in px
Height int // in px
}
// String : convert ExtraParams to string, used to generate cache path
func (e *ExtraParams) String() string {
return fmt.Sprintf("_width=%d&height=%d", e.Width, e.Height)
}
func switchProxyMode() {
matched, _ := regexp.MatchString(`^https?://`, Config.ImgPath)
if matched {
ProxyMode = true
}
}

113
encoder.go → encoder/encoder.go
View File

@ -1,18 +1,35 @@
package main
package encoder
import (
"errors"
"os"
"path"
"path/filepath"
"runtime"
"strings"
"sync"
"webp_server_go/config"
"webp_server_go/helper"
"github.com/davidbyttow/govips/v2/vips"
log "github.com/sirupsen/logrus"
)
func resizeImage(img *vips.ImageRef, extraParams ExtraParams) error {
var (
boolFalse vips.BoolParameter
intMinusOne vips.IntParameter
)
func init() {
vips.Startup(&vips.Config{
ConcurrencyLevel: runtime.NumCPU(),
})
boolFalse.Set(false)
intMinusOne.Set(-1)
}
func resizeImage(img *vips.ImageRef, extraParams config.ExtraParams) error {
imgHeightWidthRatio := float32(img.Metadata().Height) / float32(img.Metadata().Width)
if extraParams.Width > 0 && extraParams.Height > 0 {
err := img.Thumbnail(extraParams.Width, extraParams.Height, 0)
@ -33,12 +50,12 @@ func resizeImage(img *vips.ImageRef, extraParams ExtraParams) error {
return nil
}
func convertFilter(raw, avifPath string, webpPath string, extraParams ExtraParams, c chan int) {
func ConvertFilter(raw, avifPath, webpPath string, extraParams config.ExtraParams, c chan int) {
// all absolute paths
var wg sync.WaitGroup
wg.Add(2)
if !imageExists(avifPath) && config.EnableAVIF {
if !helper.ImageExists(avifPath) && config.Config.EnableAVIF {
go func() {
err := convertImage(raw, avifPath, "avif", extraParams)
if err != nil {
@ -50,7 +67,7 @@ func convertFilter(raw, avifPath string, webpPath string, extraParams ExtraParam
wg.Done()
}
if !imageExists(webpPath) {
if !helper.ImageExists(webpPath) {
go func() {
err := convertImage(raw, webpPath, "webp", extraParams)
if err != nil {
@ -68,11 +85,11 @@ func convertFilter(raw, avifPath string, webpPath string, extraParams ExtraParam
}
}
func convertImage(raw, optimized, itype string, extraParams ExtraParams) error {
func convertImage(raw, optimized, imageType string, extraParams config.ExtraParams) error {
// we don't have /path/to/tsuki.jpg.1582558990.webp, maybe we have /path/to/tsuki.jpg.1082008000.webp
// delete the old converted pic and convert a new one.
// optimized: /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
// we'll delete file starts with /home/webp_server/exhaust/path/to/tsuki.jpg.ts.itype
// we'll delete file starts with /home/webp_server/exhaust/path/to/tsuki.jpg.ts.imageType
// If contain extraParams like tsuki.jpg?width=200, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=200
s := strings.Split(path.Base(optimized), ".")
@ -93,20 +110,33 @@ func convertImage(raw, optimized, itype string, extraParams ExtraParams) error {
log.Error(err.Error())
}
switch itype {
switch imageType {
case "webp":
err = webpEncoder(raw, optimized, config.Quality, extraParams)
err = webpEncoder(raw, optimized, extraParams)
case "avif":
err = avifEncoder(raw, optimized, config.Quality, extraParams)
err = avifEncoder(raw, optimized, extraParams)
}
return err
}
func avifEncoder(p1, p2 string, quality int, extraParams ExtraParams) error {
func imageIgnore(imageFormat vips.ImageType) bool {
// Ignore Unknown, WebP, AVIF
ignoreList := []vips.ImageType{vips.ImageTypeUnknown, vips.ImageTypeWEBP, vips.ImageTypeAVIF}
for _, ignore := range ignoreList {
if imageFormat == ignore {
// Return err to render original image
return true
}
}
return false
}
func avifEncoder(p1, p2 string, extraParams config.ExtraParams) error {
// if convert fails, return error; success nil
var buf []byte
var boolFalse vips.BoolParameter
boolFalse.Set(false)
var (
buf []byte
quality = config.Config.Quality
)
img, err := vips.LoadImageFromFile(p1, &vips.ImportParams{
FailOnError: boolFalse,
})
@ -114,18 +144,11 @@ func avifEncoder(p1, p2 string, quality int, extraParams ExtraParams) error {
return err
}
// Ignore Unknown, WebP, AVIF
ignoreList := []vips.ImageType{vips.ImageTypeUnknown, vips.ImageTypeWEBP, vips.ImageTypeAVIF}
imageFormat := img.Format()
for _, ignore := range ignoreList {
if imageFormat == ignore {
// Return err to render original image
return errors.New("encoder: ignore image type")
}
if imageIgnore(img.Format()) {
return errors.New("encoder: ignore image type")
}
if config.EnableExtraParams {
if config.Config.EnableExtraParams {
err = resizeImage(img, extraParams)
if err != nil {
return err
@ -133,7 +156,7 @@ func avifEncoder(p1, p2 string, quality int, extraParams ExtraParams) error {
}
// AVIF has a maximum resolution of 65536 x 65536 pixels.
if img.Metadata().Width > avifMax || img.Metadata().Height > avifMax {
if img.Metadata().Width > config.AvifMax || img.Metadata().Height > config.AvifMax {
return errors.New("AVIF: image too large")
}
@ -171,13 +194,13 @@ func avifEncoder(p1, p2 string, quality int, extraParams ExtraParams) error {
return nil
}
func webpEncoder(p1, p2 string, quality int, extraParams ExtraParams) error {
func webpEncoder(p1, p2 string, extraParams config.ExtraParams) error {
// if convert fails, return error; success nil
var buf []byte
var boolFalse vips.BoolParameter
boolFalse.Set(false)
var intMinusOne vips.IntParameter
intMinusOne.Set(-1)
var (
buf []byte
quality = config.Config.Quality
)
img, err := vips.LoadImageFromFile(p1, &vips.ImportParams{
FailOnError: boolFalse,
NumPages: intMinusOne,
@ -186,18 +209,11 @@ func webpEncoder(p1, p2 string, quality int, extraParams ExtraParams) error {
return err
}
// Ignore Unknown, WebP, AVIF
ignoreList := []vips.ImageType{vips.ImageTypeUnknown, vips.ImageTypeWEBP, vips.ImageTypeAVIF}
imageFormat := img.Format()
for _, ignore := range ignoreList {
if imageFormat == ignore {
// Return err to render original image
return errors.New("encoder: ignore image type")
}
if imageIgnore(img.Format()) {
return errors.New("encoder: ignore image type")
}
if config.EnableExtraParams {
if config.Config.EnableExtraParams {
err = resizeImage(img, extraParams)
if err != nil {
return err
@ -205,8 +221,7 @@ func webpEncoder(p1, p2 string, quality int, extraParams ExtraParams) error {
}
// The maximum pixel dimensions of a WebP image is 16383 x 16383.
// But GIF is exception, it can be larger than 16383
if (img.Metadata().Width > webpMax || img.Metadata().Height > webpMax) && imageFormat != vips.ImageTypeGIF {
if (img.Metadata().Width > config.WebpMax || img.Metadata().Height > config.WebpMax) && img.Format() != vips.ImageTypeGIF {
return errors.New("WebP: image too large")
}
@ -218,14 +233,16 @@ func webpEncoder(p1, p2 string, quality int, extraParams ExtraParams) error {
// If quality >= 100, we use lossless mode
if quality >= 100 {
buf, _, err = img.ExportWebp(&vips.WebpExportParams{
Lossless: true,
StripMetadata: true,
Lossless: true,
StripMetadata: true,
ReductionEffort: 2,
})
} else {
buf, _, err = img.ExportWebp(&vips.WebpExportParams{
Quality: quality,
Lossless: false,
StripMetadata: true,
Quality: quality,
Lossless: false,
StripMetadata: true,
ReductionEffort: 2,
})
}

24
prefetch.go → encoder/prefetch.go
View File

@ -1,4 +1,4 @@
package main
package encoder
import (
"fmt"
@ -7,24 +7,26 @@ import (
"path/filepath"
"strings"
"time"
"webp_server_go/config"
"webp_server_go/helper"
"github.com/schollz/progressbar/v3"
log "github.com/sirupsen/logrus"
)
func prefetchImages(confImgPath string, ExhaustPath string) {
func PrefetchImages() {
// maximum ongoing prefetch is depending on your core of CPU
var sTime = time.Now()
log.Infof("Prefetching using %d cores", jobs)
var finishChan = make(chan int, jobs)
for i := 0; i < jobs; i++ {
log.Infof("Prefetching using %d cores", config.Jobs)
var finishChan = make(chan int, config.Jobs)
for i := 0; i < config.Jobs; i++ {
finishChan <- 1
}
//prefetch, recursive through the dir
all := fileCount(confImgPath)
all := helper.FileCount(config.Config.ImgPath)
var bar = progressbar.Default(all, "Prefetching...")
err := filepath.Walk(confImgPath,
err := filepath.Walk(config.Config.ImgPath,
func(picAbsPath string, info os.FileInfo, err error) error {
if err != nil {
return err
@ -33,11 +35,11 @@ func prefetchImages(confImgPath string, ExhaustPath string) {
return nil
}
// RawImagePath string, ImgFilename string, reqURI string
proposedURI := strings.Replace(picAbsPath, confImgPath, "", 1)
avif, webp := genOptimizedAbsPath(picAbsPath, ExhaustPath, info.Name(), proposedURI, ExtraParams{Width: 0, Height: 0})
proposedURI := strings.Replace(picAbsPath, config.Config.ImgPath, "", 1)
avif, webp := helper.GenOptimizedAbsPath(picAbsPath, proposedURI, config.ExtraParams{Width: 0, Height: 0})
_ = os.MkdirAll(path.Dir(avif), 0755)
log.Infof("Prefetching %s", picAbsPath)
go convertFilter(picAbsPath, avif, webp, ExtraParams{Width: 0, Height: 0}, finishChan)
go ConvertFilter(picAbsPath, avif, webp, config.ExtraParams{Width: 0, Height: 0}, finishChan)
_ = bar.Add(<-finishChan)
return nil
})
@ -46,6 +48,6 @@ func prefetchImages(confImgPath string, ExhaustPath string) {
log.Errorln(err)
}
elapsed := time.Since(sTime)
_, _ = fmt.Fprintf(os.Stdout, "Prefetch completeY(^_^)Y in %s\n\n", elapsed)
_, _ = fmt.Fprintf(os.Stdout, "Prefetch complete in %s\n\n", elapsed)
}

135
encoder_test.go
View File

@ -1,135 +0,0 @@
package main
import (
"github.com/davidbyttow/govips/v2/vips"
"github.com/stretchr/testify/assert"
"os"
"path"
"path/filepath"
"strings"
"testing"
)
var dest = "/tmp/test-result"
func walker() []string {
var list []string
_ = filepath.Walk("./pics", func(p string, info os.FileInfo, err error) error {
if !info.IsDir() && !strings.HasPrefix(path.Base(p), ".") {
list = append(list, p)
}
return nil
})
return list
}
func TestResizeImage(t *testing.T) {
// Create a test image with specific dimensions
testImage, _ := vips.NewImageFromFile("./pics/png.jpg")
// Test case 1: Both width and height are greater than 0
params1 := ExtraParams{
Width: 100,
Height: 100,
}
err := resizeImage(testImage, params1)
if err != nil {
t.Errorf("Error occurred while resizing image: %v", err)
}
// Assert the resized image has the expected dimensions
resizedWidth1 := testImage.Width()
resizedHeight1 := testImage.Height()
expectedWidth1 := params1.Width
expectedHeight1 := params1.Height
// If both width and height are provided, follow Width and keep aspect ratio
if resizedWidth1 != expectedWidth1 {
t.Errorf("Resized image dimensions do not match. Expected: %dx%d, Actual: %dx%d",
expectedWidth1, expectedHeight1, resizedWidth1, resizedHeight1)
}
// Test case 2: Only width is greater than 0
params2 := ExtraParams{
Width: 100,
Height: 0,
}
err = resizeImage(testImage, params2)
if err != nil {
t.Errorf("Error occurred while resizing image: %v", err)
}
// Assert the resized image has the expected width
resizedWidth2 := testImage.Width()
expectedWidth2 := params2.Width
if resizedWidth2 != expectedWidth2 {
t.Errorf("Resized image width does not match. Expected: %d, Actual: %d",
expectedWidth2, resizedWidth2)
}
// Test case 3: Only height is greater than 0
params3 := ExtraParams{
Width: 0,
Height: 100,
}
err = resizeImage(testImage, params3)
if err != nil {
t.Errorf("Error occurred while resizing image: %v", err)
}
// Assert the resized image has the expected height
resizedHeight3 := testImage.Height()
expectedHeight3 := params3.Height
if resizedHeight3 != expectedHeight3 {
t.Errorf("Resized image height does not match. Expected: %d, Actual: %d",
expectedHeight3, resizedHeight3)
}
}
func TestWebPEncoder(t *testing.T) {
// Go through every files
var target = walker()
for _, f := range target {
runEncoder(t, f, dest)
}
_ = os.Remove(dest)
}
func TestAnimatedGIFWithWebPEncoder(t *testing.T) {
runEncoder(t, "./pics/gif-animated.gif", dest)
_ = os.Remove(dest)
runEncoder(t, "./pics/no.gif", dest)
_ = os.Remove(dest)
}
func TestAvifEncoder(t *testing.T) {
// Only one file: img_over_16383px.jpg might cause memory issues on CI environment
assert.Nil(t, avifEncoder("./pics/big.jpg", dest, 80, ExtraParams{Width: 0, Height: 0}))
assertType(t, dest, "image/avif")
}
func TestNonExistImage(t *testing.T) {
assert.NotNil(t, webpEncoder("./pics/empty.jpg", dest, 80, ExtraParams{Width: 0, Height: 0}))
assert.NotNil(t, avifEncoder("./pics/empty.jpg", dest, 80, ExtraParams{Width: 0, Height: 0}))
}
func TestHighResolutionImage(t *testing.T) {
assert.NotNil(t, webpEncoder("./pics/img_over_16383px.jpg", dest, 80, ExtraParams{Width: 0, Height: 0}))
assert.Nil(t, avifEncoder("./pics/img_over_16383px.jpg", dest, 80, ExtraParams{Width: 0, Height: 0}))
}
func runEncoder(t *testing.T, file string, dest string) {
if file == "pics/empty.jpg" {
t.Log("Empty file, that's okay.")
}
_ = webpEncoder(file, dest, 80, ExtraParams{Width: 0, Height: 0})
assertType(t, dest, "image/webp")
}
func assertType(t *testing.T, dest, mime string) {
data, _ := os.ReadFile(dest)
types := getFileContentType(data[:512])
assert.Equalf(t, mime, types, "File %s should be %s", dest, mime)
}

27
go.mod
View File

@ -3,32 +3,35 @@ module webp_server_go
go 1.20
require (
github.com/cespare/xxhash v1.1.0
github.com/davidbyttow/govips/v2 v2.13.0
github.com/gofiber/fiber/v2 v2.4.0
github.com/gofiber/fiber/v2 v2.46.0
github.com/h2non/filetype v1.1.3
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/schollz/progressbar/v3 v3.13.1
github.com/sirupsen/logrus v1.9.3
github.com/stretchr/testify v1.8.4
github.com/valyala/fasthttp v1.47.0
)
require (
github.com/andybalholm/brotli v1.0.5 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/klauspost/compress v1.16.3 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.18 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rivo/uniseg v0.4.3 // indirect
github.com/philhofer/fwd v1.1.2 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 // indirect
github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee // indirect
github.com/stretchr/testify v1.8.4 // indirect
github.com/tinylib/msgp v1.1.8 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/tcplisten v1.0.0 // indirect
golang.org/x/image v0.7.0 // indirect
golang.org/x/image v0.5.0 // indirect
golang.org/x/net v0.8.0 // indirect
golang.org/x/sys v0.7.0 // indirect
golang.org/x/term v0.7.0 // indirect
golang.org/x/text v0.9.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
golang.org/x/sys v0.8.0 // indirect
golang.org/x/term v0.6.0 // indirect
golang.org/x/text v0.8.0 // indirect
)
replace github.com/gofiber/fiber/v2 v2.4.0 => github.com/webp-sh/fiber/v2 v2.4.0

72
go.sum
View File

@ -1,109 +1,131 @@
github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davidbyttow/govips/v2 v2.13.0 h1:5MK9ZcXZC5GzUR9Ca8fJwOYqMgll/H096ec0PJP59QM=
github.com/davidbyttow/govips/v2 v2.13.0/go.mod h1:LPTrwWtNa5n4yl9UC52YBOEGdZcY5hDTP4Ms2QWasTw=
github.com/gofiber/fiber/v2 v2.46.0 h1:wkkWotblsGVlLjXj2dpgKQAYHtXumsK/HyFugQM68Ns=
github.com/gofiber/fiber/v2 v2.46.0/go.mod h1:DNl0/c37WLe0g92U6lx1VMQuxGUQY5V7EIaVoEsUffc=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY=
github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98=
github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw=
github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw=
github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 h1:rmMl4fXJhKMNWl+K+r/fq4FbbKI+Ia2m9hYBLm2h4G4=
github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94/go.mod h1:90zrgN3D/WJsDd1iXHT96alCoN2KJo6/4x1DZC3wZs8=
github.com/savsgio/gotils v0.0.0-20220530130905-52f3993e8d6d/go.mod h1:Gy+0tqhJvgGlqnTF8CVGP0AaGRjwBtXs/a5PA0Y3+A4=
github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee h1:8Iv5m6xEo1NR1AvpV+7XmhI4r39LGNzwUL4YpMuL5vk=
github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee/go.mod h1:qwtSXrKuJh/zsFQ12yEE89xfCrGKK63Rr7ctU/uCo4g=
github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE=
github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw=
github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0=
github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.18.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8ermTRJNroD7A=
github.com/valyala/fasthttp v1.47.0 h1:y7moDoxYzMooFpT5aHgNgVOQDrS3qlkfiP9mDtGGK9c=
github.com/valyala/fasthttp v1.47.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
github.com/webp-sh/fiber/v2 v2.4.0 h1:JtkW0HAqHCExodZMZnG7GrLiJuK2YbNYw8eXo55+tr8=
github.com/webp-sh/fiber/v2 v2.4.0/go.mod h1:f8BRRIMjMdRyt2qmJ/0Sea3j3rwwfufPrh9WNBRiVZ0=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/image v0.5.0 h1:5JMiNunQeQw++mMOz48/ISeNu3Iweh/JaZU8ZLqHRrI=
golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4=
golang.org/x/image v0.7.0 h1:gzS29xtG1J5ybQlv0PuyfE3nmc6R4qB73m6LUUmvFuw=
golang.org/x/image v0.7.0/go.mod h1:nd/q4ef1AKKYl/4kft7g+6UyGbdiqWqTP1ZAbRoV7Rg=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201210223839-7e3030f88018/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw=
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ=
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U=
gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

123
handler/remote.go Normal file
View File

@ -0,0 +1,123 @@
package handler
import (
"bytes"
"net/http"
"os"
"path"
"path/filepath"
"strings"
"webp_server_go/config"
"webp_server_go/helper"
"github.com/gofiber/fiber/v2"
"github.com/h2non/filetype"
log "github.com/sirupsen/logrus"
)
// Given /path/to/node.png
// Delete /path/to/node.png*
func cleanProxyCache(cacheImagePath string) {
// Delete /node.png*
files, err := filepath.Glob(cacheImagePath + "*")
if err != nil {
log.Infoln(err)
}
for _, f := range files {
if err := os.Remove(f); err != nil {
log.Info(err)
}
}
}
func downloadFile(filepath string, url string) {
resp, err := http.Get(url)
if err != nil {
log.Errorln("Connection to remote error when downloadFile!")
return
}
defer resp.Body.Close()
if resp.StatusCode != fiber.StatusOK {
log.Errorf("remote returned %s when fetching remote image", resp.Status)
return
}
// Copy bytes here
bodyBytes := new(bytes.Buffer)
_, err = bodyBytes.ReadFrom(resp.Body)
if err != nil {
return
}
// Check if remote content-type is image using check by filetype instead of content-type returned by origin
kind, _ := filetype.Match(bodyBytes.Bytes())
mime := kind.MIME.Value
if !strings.Contains(mime, "image") {
log.Errorf("remote file %s is not image, remote content has MIME type of %s", url, mime)
return
}
_ = os.MkdirAll(path.Dir(filepath), 0755)
// Create Cache here as a lock, so we can prevent incomplete file from being read
// Key: filepath, Value: true
config.WriteLock.Set(filepath, true, -1)
err = os.WriteFile(filepath, bodyBytes.Bytes(), 0600)
if err != nil {
// not likely to happen
return
}
// Delete lock here
config.WriteLock.Delete(filepath)
}
func fetchRemoteImg(url string) string {
// url is https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200
// How do we know if the remote img is changed? we're using hash(url+etag) as key.
// if this exists in local system, means the remote img is not changed, we can use it directly.
// otherwise, we need to fetch it from remote and store it in local system.
log.Infof("Remote Addr is %s, pinging for info...", url)
// identifiable is etag + length
identifiable := pingURL(url)
// For store the remote raw image, /home/webp_server/remote-raw/3a42ab801f669d64-b8f999ab5acd69d03f5e904b1b84eb79210536
// Which 3a42ab801f669d64 is hash(url), b8f999ab5acd69d03f5e904b1b84eb79 is etag and 210536 is length
localRawImagePath := path.Join(config.RemoteRaw, helper.HashString(url)+"-"+identifiable)
if helper.ImageExists(localRawImagePath) {
return localRawImagePath
} else {
// Temporary store of remote file.
cleanProxyCache(config.RemoteRaw + helper.HashString(url) + "*")
log.Info("Remote file not found in remote-raw, re-fetching...")
downloadFile(localRawImagePath, url)
return localRawImagePath
}
}
func pingURL(url string) string {
// this function will try to return identifiable info, currently include etag, content-length as string
// anything goes wrong, will return ""
var etag, length string
resp, err := http.Head(url)
if err != nil {
log.Errorln("Connection to remote error when pingUrl!")
return ""
}
defer resp.Body.Close()
if resp.StatusCode == fiber.StatusOK {
etag = resp.Header.Get("etag")
length = resp.Header.Get("content-length")
}
if etag == "" {
log.Info("Remote didn't return etag in header when getRemoteImageInfo, please check.")
}
// Remove " from etag
etag = strings.ReplaceAll(etag, "\"", "")
return etag + length
}

104
handler/router.go Normal file
View File

@ -0,0 +1,104 @@
package handler
import (
"net/http"
"net/url"
"strings"
"webp_server_go/config"
"webp_server_go/encoder"
"webp_server_go/helper"
"path"
"strconv"
"github.com/gofiber/fiber/v2"
log "github.com/sirupsen/logrus"
)
func Convert(c *fiber.Ctx) error {
// this function need to do:
// 1. get request path, query string
// 2. generate rawImagePath, could be local path or remote url(possible with query string)
// 3. pass it to encoder, get the result, send it back
var (
reqURI, _ = url.QueryUnescape(c.Path()) // /mypic/123.jpg
reqURIwithQuery, _ = url.QueryUnescape(c.OriginalURL()) // /mypic/123.jpg?someother=200&somebugs=200
filename = path.Base(reqURI)
)
if !helper.CheckAllowedType(filename) {
msg := "File extension not allowed! " + filename
log.Warn(msg)
c.Status(http.StatusBadRequest)
_ = c.Send([]byte(msg))
return nil
}
// Sometimes reqURIwithQuery can be https://example.tld/mypic/123.jpg?someother=200&somebugs=200, we need to extract it.
// delete ../ in reqURI to mitigate directory traversal
reqURI = path.Clean(reqURI)
reqURIwithQuery = path.Clean(reqURIwithQuery)
WidthInt, err := strconv.Atoi(c.Query("width"))
if err != nil {
WidthInt = 0
}
HeightInt, err := strconv.Atoi(c.Query("height"))
if err != nil {
HeightInt = 0
}
var extraParams = config.ExtraParams{
Width: WidthInt,
Height: HeightInt,
}
var rawImageAbs string
if config.ProxyMode {
// this is proxyMode, we'll have to use this url to download and save it to local path, which also gives us rawImageAbs
// https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200
rawImageAbs = fetchRemoteImg(config.Config.ImgPath + reqURIwithQuery)
} else {
// not proxyMode, we'll use local path
rawImageAbs = path.Join(config.Config.ImgPath, reqURI) // /home/xxx/mypic/123.jpg
}
goodFormat := helper.GuessSupportedFormat(&c.Request().Header)
// Check the original image for existence,
if !helper.ImageExists(rawImageAbs) {
msg := "image not found"
_ = c.Send([]byte(msg))
log.Warn(msg)
_ = c.SendStatus(404)
return nil
}
// generate with timestamp to make sure files are update-to-date
// If extraParams not enabled, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
// If extraParams enabled, and given request at tsuki.jpg?width=200, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=200&height=0
// If extraParams enabled, and given request at tsuki.jpg, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=0&height=0
avifAbs, webpAbs := helper.GenOptimizedAbsPath(rawImageAbs, reqURI, extraParams)
encoder.ConvertFilter(rawImageAbs, avifAbs, webpAbs, extraParams, nil)
var availableFiles = []string{rawImageAbs}
for _, v := range goodFormat {
if v == "avif" {
availableFiles = append(availableFiles, avifAbs)
}
if v == "webp" {
availableFiles = append(availableFiles, webpAbs)
}
}
finalFilename := helper.FindSmallestFiles(availableFiles)
if strings.HasSuffix(finalFilename, ".webp ") {
c.Set("Content-Type", "image/webp")
} else if strings.HasSuffix(finalFilename, ".avif") {
c.Set("Content-Type", "image/avif")
}
c.Set("X-Compression-Rate", helper.GetCompressionRate(rawImageAbs, finalFilename))
return c.SendFile(finalFilename)
}

279
helper.go
View File

@ -1,279 +0,0 @@
package main
import (
"bytes"
"crypto/sha1" //#nosec
"encoding/hex"
"fmt"
"net/http"
"os"
"path"
"path/filepath"
"time"
"github.com/h2non/filetype"
"github.com/valyala/fasthttp"
"strings"
log "github.com/sirupsen/logrus"
)
func avifMatcher(buf []byte) bool {
// 0000001c 66747970 61766966 00000000 61766966 6d696631 6d696166
return len(buf) > 1 && bytes.Equal(buf[:28], []byte{
0x0, 0x0, 0x0, 0x1c,
0x66, 0x74, 0x79, 0x70,
0x61, 0x76, 0x69, 0x66,
0x0, 0x0, 0x0, 0x0,
0x61, 0x76, 0x69, 0x66,
0x6d, 0x69, 0x66, 0x31,
0x6d, 0x69, 0x61, 0x66,
})
}
func getFileContentType(buffer []byte) string {
// TODO deprecated.
var avifType = filetype.NewType("avif", "image/avif")
filetype.AddMatcher(avifType, avifMatcher)
kind, _ := filetype.Match(buffer)
return kind.MIME.Value
}
func fileCount(dir string) int64 {
var count int64 = 0
_ = filepath.Walk(dir,
func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() {
count += 1
}
return nil
})
return count
}
func imageExists(filename string) bool {
info, err := os.Stat(filename)
if os.IsNotExist(err) {
return false
}
if info.Size() < 100 {
// means something wrong in exhaust file system
return false
}
// Check if there is lock in cache, retry after 1 second
maxRetries := 3
retryDelay := 100 * time.Millisecond // Initial retry delay
for retry := 0; retry < maxRetries; retry++ {
if _, found := WriteLock.Get(filename); found {
log.Infof("file %s is locked, retrying in %s", filename, retryDelay)
time.Sleep(retryDelay)
retryDelay *= 2 // Exponential backoff
} else {
return !info.IsDir()
}
}
log.Debugf("file %s exists!", filename)
return !info.IsDir()
}
func checkAllowedType(imgFilename string) bool {
imgFilename = strings.ToLower(imgFilename)
for _, allowedType := range config.AllowedTypes {
if allowedType == "*" {
return true
}
allowedType = "." + strings.ToLower(allowedType)
if strings.HasSuffix(imgFilename, allowedType) {
return true
}
}
return false
}
// Check for remote filepath, e.g: https://test.webp.sh/node.png
// return StatusCode, etagValue and length
func getRemoteImageInfo(fileURL string) (int, string, string) {
resp, err := http.Head(fileURL)
if err != nil {
log.Errorln("Connection to remote error when getRemoteImageInfo!")
return http.StatusInternalServerError, "", ""
}
defer resp.Body.Close()
if resp.StatusCode == 200 {
etagValue := resp.Header.Get("etag")
if etagValue == "" {
log.Info("Remote didn't return etag in header when getRemoteImageInfo, please check.")
} else {
return resp.StatusCode, etagValue, resp.Header.Get("content-length")
}
}
return resp.StatusCode, "", resp.Header.Get("content-length")
}
func fetchRemoteImage(filepath string, url string) error {
resp, err := http.Get(url)
if err != nil {
log.Errorln("Connection to remote error when fetchRemoteImage!")
return err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return fmt.Errorf("remote returned %s when fetching remote image", resp.Status)
}
// Copy bytes here
bodyBytes := new(bytes.Buffer)
_, err = bodyBytes.ReadFrom(resp.Body)
if err != nil {
return err
}
// Check if remote content-type is image using check by filetype instead of content-type returned by origin
kind, _ := filetype.Match(bodyBytes.Bytes())
if kind == filetype.Unknown || !strings.Contains(kind.MIME.Value, "image") {
return fmt.Errorf("remote file %s is not image, remote content has MIME type of %s", url, kind.MIME.Value)
}
_ = os.MkdirAll(path.Dir(filepath), 0755)
// Create Cache here as a lock
// Key: filepath, Value: true
WriteLock.Set(filepath, true, -1)
err = os.WriteFile(filepath, bodyBytes.Bytes(), 0600)
if err != nil {
return err
}
// Delete lock here
WriteLock.Delete(filepath)
return nil
}
// Given /path/to/node.png
// Delete /path/to/node.png*
func cleanProxyCache(cacheImagePath string) {
// Delete /node.png*
files, err := filepath.Glob(cacheImagePath + "*")
if err != nil {
log.Infoln(err)
}
for _, f := range files {
if err := os.Remove(f); err != nil {
log.Info(err)
}
}
}
func genOptimizedAbsPath(rawImagePath string, exhaustPath string, imageName string, reqURI string, extraParams ExtraParams) (string, string) {
// get file mod time
STAT, err := os.Stat(rawImagePath)
if err != nil {
log.Error(err.Error())
return "", ""
}
ModifiedTime := STAT.ModTime().Unix()
// webpFilename: abc.jpg.png -> abc.jpg.png.1582558990.webp
webpFilename := fmt.Sprintf("%s.%d.webp", imageName, ModifiedTime)
// avifFilename: abc.jpg.png -> abc.jpg.png.1582558990.avif
avifFilename := fmt.Sprintf("%s.%d.avif", imageName, ModifiedTime)
// If extraParams not enabled, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
// If extraParams enabled, and given request at tsuki.jpg?width=200, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=200&height=0
// If extraParams enabled, and given request at tsuki.jpg, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=0&height=0
if config.EnableExtraParams {
webpFilename = webpFilename + extraParams.String()
avifFilename = avifFilename + extraParams.String()
}
// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
// Custom Exhaust: /path/to/exhaust/web_path/web_to/tsuki.jpg.1582558990.webp
webpAbsolutePath := path.Clean(path.Join(exhaustPath, path.Dir(reqURI), webpFilename))
avifAbsolutePath := path.Clean(path.Join(exhaustPath, path.Dir(reqURI), avifFilename))
return avifAbsolutePath, webpAbsolutePath
}
func getCompressionRate(RawImagePath string, optimizedImg string) string {
originFileInfo, err := os.Stat(RawImagePath)
if err != nil {
log.Warnf("Failed to get raw image %v", err)
return ""
}
optimizedFileInfo, err := os.Stat(optimizedImg)
if err != nil {
log.Warnf("Failed to get optimized image %v", err)
return ""
}
compressionRate := float64(optimizedFileInfo.Size()) / float64(originFileInfo.Size())
log.Debugf("The compression rate is %d/%d=%.2f", originFileInfo.Size(), optimizedFileInfo.Size(), compressionRate)
return fmt.Sprintf(`%.2f`, compressionRate)
}
func guessSupportedFormat(header *fasthttp.RequestHeader) []string {
var supported = map[string]bool{
"raw": true,
"webp": false,
"avif": false}
var ua = string(header.Peek("user-agent"))
var accept = strings.ToLower(string(header.Peek("accept")))
log.Debugf("%s\t%s\n", ua, accept)
if strings.Contains(accept, "image/webp") {
supported["webp"] = true
}
if strings.Contains(accept, "image/avif") {
supported["avif"] = true
}
// chrome on iOS will not send valid image accept header
if strings.Contains(ua, "iPhone OS 14") || strings.Contains(ua, "CPU OS 14") ||
strings.Contains(ua, "iPhone OS 15") || strings.Contains(ua, "CPU OS 15") {
supported["webp"] = true
} else if strings.Contains(ua, "Android") || strings.Contains(ua, "Linux") {
supported["webp"] = true
}
var accepted []string
for k, v := range supported {
if v {
accepted = append(accepted, k)
}
}
return accepted
}
func findSmallestFiles(files []string) string {
// walk files
var small int64
var final string
for _, f := range files {
stat, err := os.Stat(f)
if err != nil {
log.Warnf("%s not found on filesystem", f)
continue
}
if stat.Size() < small || small == 0 {
small = stat.Size()
final = f
}
}
return final
}
func Sha1Path(uri string) string {
/* #nosec */
h := sha1.New()
h.Write([]byte(uri))
return hex.EncodeToString(h.Sum(nil))
}

182
helper/helper.go Normal file
View File

@ -0,0 +1,182 @@
package helper
import (
"fmt"
"github.com/cespare/xxhash"
"github.com/valyala/fasthttp"
"os"
"path"
"path/filepath"
"strings"
"time"
"webp_server_go/config"
log "github.com/sirupsen/logrus"
)
func FileCount(dir string) int64 {
var count int64 = 0
_ = filepath.Walk(dir,
func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() {
count += 1
}
return nil
})
return count
}
func ImageExists(filename string) bool {
info, err := os.Stat(filename)
if os.IsNotExist(err) {
return false
}
// if file size is less than 100 bytes, we assume it's invalid file
// png starts with an 8-byte signature, follow by 4 chunks 58 bytes.
// JPG is 134 bytes.
// webp is 33 bytes.
if info.Size() < 100 {
// means something wrong in exhaust file system
return false
}
// Check if there is lock in cache, retry after 1 second
maxRetries := 3
retryDelay := 100 * time.Millisecond // Initial retry delay
for retry := 0; retry < maxRetries; retry++ {
if _, found := config.WriteLock.Get(filename); found {
log.Infof("file %s is locked, retrying in %s", filename, retryDelay)
time.Sleep(retryDelay)
retryDelay *= 2 // Exponential backoff
} else {
return !info.IsDir()
}
}
return !info.IsDir()
}
func CheckAllowedType(imgFilename string) bool {
for _, allowedType := range config.Config.AllowedTypes {
if allowedType == "*" {
return true
}
allowedType = "." + strings.ToLower(allowedType)
if strings.HasSuffix(strings.ToLower(imgFilename), allowedType) {
return true
}
}
return false
}
func GenOptimizedAbsPath(rawImagePath, reqURI string, extraParams config.ExtraParams) (string, string) {
// imageName is not needed, we can use reqURI
// get file mod time
var (
imageName = path.Base(reqURI)
exhaustPath = config.Config.ExhaustPath
)
STAT, err := os.Stat(rawImagePath)
if err != nil {
log.Error(err.Error())
return "", ""
}
ModifiedTime := STAT.ModTime().Unix()
// TODO: just hash it?
// webpFilename: abc.jpg.png -> abc.jpg.png.1582558990.webp
webpFilename := fmt.Sprintf("%s.%d.webp", imageName, ModifiedTime)
// avifFilename: abc.jpg.png -> abc.jpg.png.1582558990.avif
avifFilename := fmt.Sprintf("%s.%d.avif", imageName, ModifiedTime)
// If extraParams not enabled, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
// If extraParams enabled, and given request at tsuki.jpg?width=200, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=200&height=0
// If extraParams enabled, and given request at tsuki.jpg, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=0&height=0
if config.Config.EnableExtraParams {
webpFilename = webpFilename + extraParams.String()
avifFilename = avifFilename + extraParams.String()
}
// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
// Custom Exhaust: /path/to/exhaust/web_path/web_to/tsuki.jpg.1582558990.webp
webpAbsolutePath := path.Clean(path.Join(exhaustPath, path.Dir(reqURI), webpFilename))
avifAbsolutePath := path.Clean(path.Join(exhaustPath, path.Dir(reqURI), avifFilename))
return avifAbsolutePath, webpAbsolutePath
}
func GetCompressionRate(RawImagePath string, optimizedImg string) string {
originFileInfo, err := os.Stat(RawImagePath)
if err != nil {
log.Warnf("Failed to get raw image %v", err)
return ""
}
optimizedFileInfo, err := os.Stat(optimizedImg)
if err != nil {
log.Warnf("Failed to get optimized image %v", err)
return ""
}
compressionRate := float64(optimizedFileInfo.Size()) / float64(originFileInfo.Size())
return fmt.Sprintf(`%.2f`, compressionRate)
}
func GuessSupportedFormat(header *fasthttp.RequestHeader) []string {
var (
supported = map[string]bool{
"raw": true,
"webp": false,
"avif": false,
}
ua = string(header.Peek("user-agent"))
accept = strings.ToLower(string(header.Peek("accept")))
)
if strings.Contains(accept, "image/webp") {
supported["webp"] = true
}
if strings.Contains(accept, "image/avif") {
supported["avif"] = true
}
// chrome on iOS will not send valid image accept header
if strings.Contains(ua, "iPhone OS 14") || strings.Contains(ua, "CPU OS 14") ||
strings.Contains(ua, "iPhone OS 15") || strings.Contains(ua, "CPU OS 15") ||
strings.Contains(ua, "Android") || strings.Contains(ua, "Linux") {
supported["webp"] = true
}
// save true value's key to slice
var accepted []string
for k, v := range supported {
if v {
accepted = append(accepted, k)
}
}
return accepted
}
func FindSmallestFiles(files []string) string {
// walk files
var small int64
var final string
for _, f := range files {
stat, err := os.Stat(f)
if err != nil {
log.Warnf("%s not found on filesystem", f)
continue
}
if stat.Size() < small || small == 0 {
small = stat.Size()
final = f
}
}
return final
}
func HashString(uri string) string {
// xxhash supports cross compile
return fmt.Sprintf("%x", xxhash.Sum64String(uri))
}

237
helper_test.go
View File

@ -1,237 +0,0 @@
package main
import (
"fmt"
"path"
"time"
log "github.com/sirupsen/logrus"
"github.com/valyala/fasthttp"
"net/http"
"os"
"path/filepath"
"sort"
"testing"
"github.com/stretchr/testify/assert"
)
// test all files: go test -v -cover .
// test one case: go test -v -run TestSelectFormat
func TestGetFileContentType(t *testing.T) {
var data = []byte("remember remember the 5th of november")
var expected = ""
var result = getFileContentType(data)
assert.Equalf(t, result, expected, "Result: [%s], Expected: [%s]", result, expected)
}
func TestFileCount(t *testing.T) {
var data = "scripts"
var expected int64 = 2
var result = fileCount(data)
assert.Equalf(t, result, expected, "Result: [%d], Expected: [%d]", result, expected)
}
func TestImageExists(t *testing.T) {
var data = "./pics/empty.jpg"
var result = imageExists(data)
if result {
t.Errorf("Result: [%v], Expected: [%v]", result, false)
}
data = ".pics/empty2.jpg"
result = imageExists(data)
assert.Falsef(t, result, "Result: [%v], Expected: [%v]", result, false)
}
func TestGenOptimizedAbsPath(t *testing.T) {
// Create a temporary file for testing
tempFile, err := os.CreateTemp("", "test_image.*")
if err != nil {
t.Fatalf("Failed to create temporary file: %v", err)
}
defer os.Remove(tempFile.Name())
// Set the modification time for the temporary file
modTime := time.Now()
if err := os.Chtimes(tempFile.Name(), modTime, modTime); err != nil {
t.Fatalf("Failed to set modification time for the temporary file: %v", err)
}
rawImagePath := tempFile.Name()
exhaustPath := "/path/to/exhaust"
imageName := "tsuki.jpg"
reqURI := "/path/to/tsuki.jpg"
extraParams := ExtraParams{Width: 200, Height: 0}
// Test if config.EnableExtraParams is false
config.EnableExtraParams = false
avifAbsolutePath, webpAbsolutePath := genOptimizedAbsPath(rawImagePath, exhaustPath, imageName, reqURI, extraParams)
expectedAvifPath := path.Clean(path.Join(exhaustPath, path.Dir(reqURI), fmt.Sprintf("%s.%d.avif", imageName, modTime.Unix())))
expectedWebpPath := path.Clean(path.Join(exhaustPath, path.Dir(reqURI), fmt.Sprintf("%s.%d.webp", imageName, modTime.Unix())))
if avifAbsolutePath != expectedAvifPath {
t.Errorf("Avif absolute path is incorrect. Expected: %s, Got: %s", expectedAvifPath, avifAbsolutePath)
}
if webpAbsolutePath != expectedWebpPath {
t.Errorf("Webp absolute path is incorrect. Expected: %s, Got: %s", expectedWebpPath, webpAbsolutePath)
}
// Test if config.EnableExtraParams is true and extraParams is not 0
config.EnableExtraParams = true
avifAbsolutePath, webpAbsolutePath = genOptimizedAbsPath(rawImagePath, exhaustPath, imageName, reqURI, extraParams)
expectedAvifPath = path.Clean(path.Join(exhaustPath, path.Dir(reqURI), fmt.Sprintf("%s.%d.avif_width=%d&height=%d", imageName, modTime.Unix(), extraParams.Width, extraParams.Height)))
expectedWebpPath = path.Clean(path.Join(exhaustPath, path.Dir(reqURI), fmt.Sprintf("%s.%d.webp_width=%d&height=%d", imageName, modTime.Unix(), extraParams.Width, extraParams.Height)))
if avifAbsolutePath != expectedAvifPath {
t.Errorf("Avif absolute path is incorrect. Expected: %s, Got: %s", expectedAvifPath, avifAbsolutePath)
}
if webpAbsolutePath != expectedWebpPath {
t.Errorf("Webp absolute path is incorrect. Expected: %s, Got: %s", expectedWebpPath, webpAbsolutePath)
}
// Test if config.EnableExtraParams is true and extraParams is 0
config.EnableExtraParams = true
extraParams = ExtraParams{Width: 200, Height: 0}
avifAbsolutePath, webpAbsolutePath = genOptimizedAbsPath(rawImagePath, exhaustPath, imageName, reqURI, extraParams)
expectedAvifPath = path.Clean(path.Join(exhaustPath, path.Dir(reqURI), fmt.Sprintf("%s.%d.avif_width=%d&height=%d", imageName, modTime.Unix(), extraParams.Width, extraParams.Height)))
expectedWebpPath = path.Clean(path.Join(exhaustPath, path.Dir(reqURI), fmt.Sprintf("%s.%d.webp_width=%d&height=%d", imageName, modTime.Unix(), extraParams.Width, extraParams.Height)))
if avifAbsolutePath != expectedAvifPath {
t.Errorf("Avif absolute path is incorrect. Expected: %s, Got: %s", expectedAvifPath, avifAbsolutePath)
}
if webpAbsolutePath != expectedWebpPath {
t.Errorf("Webp absolute path is incorrect. Expected: %s, Got: %s", expectedWebpPath, webpAbsolutePath)
}
}
func TestSelectFormat(t *testing.T) {
// this is a complete test case for webp compatibility
// func goOrigin(header, ua string) bool
// UNLESS YOU KNOW WHAT YOU ARE DOING, DO NOT CHANGE THE TEST CASE MAPPING HERE.
var fullSupport = []string{"avif", "webp", "raw"}
var webpSupport = []string{"webp", "raw"}
var jpegSupport = []string{"raw"}
var testCase = map[[2]string][]string{
// Latest Chrome on Windows, macOS, linux, Android and iOS 13
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36"}: fullSupport,
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36"}: fullSupport,
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36"}: fullSupport,
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", "Mozilla/5.0 (Linux; Android 10) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.60 Mobile Safari/537.36"}: fullSupport,
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", "Mozilla/5.0 (Linux; Android 6.0; HTC M8t) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.74 Mobile Safari/537.36"}: fullSupport,
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", "HTCM8t_LTE/1.0 Android/4.4 release/2013 Browser/WAP2.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 AppleWebKit/534.30"}: webpSupport,
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 13_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/83.0.4103.63 Mobile/15E148 Safari/604.1"}: jpegSupport,
// macOS Catalina Safari
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.1 Safari/605.1.15"}: jpegSupport,
// iOS14 Safari and Chrome
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 14_2_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.1 Mobile/15E148 Safari/604.1"}: webpSupport,
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 14_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/87.0.4280.77 Mobile/15E148 Safari/604.1"}: webpSupport,
// iPadOS 14 Safari and Chrome
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPad; CPU OS 14_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.1 Mobile/15E148 Safari/604.1"}: webpSupport,
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPad; CPU OS 14_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/87.0.4280.77 Mobile/15E148 Safari/604.1"}: webpSupport,
// iOS 15 Safari, Firefox and Chrome
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 15_1_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Mobile/15E148 Safari/604.1"}: webpSupport,
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 15_1_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/39.0 Mobile/15E148 Safari/605.1.15"}: webpSupport,
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 15_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/96.0.4664.53 Mobile/15E148 Safari/604.1"}: webpSupport,
// IE
[2]string{"application/x-ms-application, image/jpeg, application/xaml+xml, image/gif, image/pjpeg, application/x-ms-xbap, */*", "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko"}: jpegSupport,
// Others
[2]string{"", "PostmanRuntime/7.26.1"}: jpegSupport,
[2]string{"", "curl/7.64.1"}: jpegSupport,
[2]string{"image/webp", "curl/7.64.1"}: webpSupport,
[2]string{"image/avif,image/webp", "curl/7.64.1"}: fullSupport,
// some weird browsers
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 15_1_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 MicroMessenger/8.0.16(0x18001033) NetType/WIFI Language/zh_CN"}: webpSupport,
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", "Mozilla/5.0 (Linux; Android 6.0; HTC M8t Build/MRA58K; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/45.0.2454.95 Mobile Safari/537.36 MMWEBID/4285 MicroMessenger/8.0.15.2001(0x28000F41) Process/tools WeChat/arm32 Weixin GPVersion/1 NetType/WIFI Language/zh_CN ABI/arm32"}: webpSupport,
}
for browser, expected := range testCase {
var header fasthttp.RequestHeader
header.Set("accept", browser[0])
header.Set("user-agent", browser[1])
guessed := guessSupportedFormat(&header)
sort.Strings(expected)
sort.Strings(guessed)
log.Infof("%s expected%s --- actual %s", browser, expected, guessed)
assert.Equal(t, expected, guessed)
}
}
func TestGetRemoteImageInfo(t *testing.T) {
url := "https://github.com/favicon.ico"
statusCode, etag, length := getRemoteImageInfo(url)
assert.NotEqual(t, "", etag)
assert.NotEqual(t, "0", length)
assert.Equal(t, statusCode, http.StatusOK)
// test non-exist url
url = "http://sdahjajda.com"
statusCode, etag, length = getRemoteImageInfo(url)
assert.Equal(t, "", etag)
assert.Equal(t, "", length)
assert.Equal(t, statusCode, http.StatusInternalServerError)
}
func TestFetchRemoteImage(t *testing.T) {
// test the normal one
fp := filepath.Join("./exhaust", "test.ico")
err := fetchRemoteImage(fp, "http://github.com/favicon.ico")
assert.Equal(t, err, nil)
data, _ := os.ReadFile(fp)
assert.Equal(t, "image/vnd.microsoft.icon", getFileContentType(data))
// test can't create file
err = fetchRemoteImage("/", "http://github.com/favicon.ico")
assert.NotNil(t, err)
// test bad url
err = fetchRemoteImage(fp, "http://ahjdsgdsghja.cya")
assert.NotNil(t, err)
}
func TestCleanProxyCache(t *testing.T) {
// test normal situation
fp := filepath.Join("./exhaust", "sample.png.12345.webp")
buf := make([]byte, 0x1000)
_ = os.WriteFile(fp, buf, 0755)
assert.True(t, imageExists(fp))
cleanProxyCache(fp)
assert.False(t, imageExists(fp))
// test bad dir
cleanProxyCache("/aasdyg/dhj2/dagh")
}
func TestGetCompressionRate(t *testing.T) {
pic1 := "pics/webp_server.bmp"
pic2 := "pics/webp_server.jpg"
var ratio string
ratio = getCompressionRate(pic1, pic2)
assert.Equal(t, "0.16", ratio)
ratio = getCompressionRate(pic1, "pic2")
assert.Equal(t, "", ratio)
ratio = getCompressionRate("pic1", pic2)
assert.Equal(t, "", ratio)
}

0
tests/glib_malloc/docker-compose.yml → malloc_tests/glib_malloc/docker-compose.yml
View File

0
tests/jemalloc/docker-compose.yml → malloc_tests/jemalloc/docker-compose.yml
View File

22
prefetch_test.go
View File

@ -1,22 +0,0 @@
package main
import (
"os"
"testing"
"github.com/stretchr/testify/assert"
)
func TestPrefetchImages(t *testing.T) {
fp := "./prefetch"
_ = os.Mkdir(fp, 0755)
prefetchImages("./pics/dir1/", "./prefetch")
count := fileCount("./prefetch")
assert.Equal(t, int64(1), count)
_ = os.RemoveAll(fp)
}
func TestBadPrefetch(t *testing.T) {
jobs = 1
prefetchImages("./pics2", "./prefetch")
}

145
router.go
View File

@ -1,145 +0,0 @@
package main
import (
"errors"
"fmt"
"net/http"
"net/url"
"path"
"strconv"
"github.com/gofiber/fiber/v2"
log "github.com/sirupsen/logrus"
)
func convert(c *fiber.Ctx) error {
//basic vars
var (
reqURI, _ = url.QueryUnescape(c.Path()) // /mypic/123.jpg
reqURIwithQuery, _ = url.QueryUnescape(c.OriginalURL()) // /mypic/123.jpg?someother=200&somebugs=200
imgFilename = path.Base(reqURI) // pure filename, 123.jpg
)
// Sometimes reqURIwithQuery can be https://example.tld/mypic/123.jpg?someother=200&somebugs=200, we need to extract it.
u, err := url.Parse(reqURIwithQuery)
if err != nil {
log.Errorln(err)
}
reqURIwithQuery = u.RequestURI()
// delete ../ in reqURI to mitigate directory traversal
reqURI = path.Clean(reqURI)
reqURIwithQuery = path.Clean(reqURIwithQuery)
// Begin Extra params
var extraParams ExtraParams
Width := c.Query("width")
Height := c.Query("height")
WidthInt, err := strconv.Atoi(Width)
if err != nil {
WidthInt = 0
}
HeightInt, err := strconv.Atoi(Height)
if err != nil {
HeightInt = 0
}
extraParams = ExtraParams{
Width: WidthInt,
Height: HeightInt,
}
// End Extra params
var rawImageAbs string
if proxyMode {
rawImageAbs = config.ImgPath + reqURIwithQuery // https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200
} else {
rawImageAbs = path.Join(config.ImgPath, reqURI) // /home/xxx/mypic/123.jpg
}
log.Debugf("Incoming connection from %s %s", c.IP(), imgFilename)
if !checkAllowedType(imgFilename) {
msg := "File extension not allowed! " + imgFilename
log.Warn(msg)
c.Status(http.StatusBadRequest)
_ = c.Send([]byte(msg))
return nil
}
goodFormat := guessSupportedFormat(&c.Request().Header)
if proxyMode {
rawImageAbs, _ = proxyHandler(c, reqURIwithQuery)
}
log.Debugf("rawImageAbs=%s", rawImageAbs)
// Check the original image for existence,
if !imageExists(rawImageAbs) {
msg := "image not found"
_ = c.Send([]byte(msg))
log.Warn(msg)
_ = c.SendStatus(404)
return nil
}
// generate with timestamp to make sure files are update-to-date
// If extraParams not enabled, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
// If extraParams enabled, and given request at tsuki.jpg?width=200, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=200&height=0
// If extraParams enabled, and given request at tsuki.jpg, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=0&height=0
avifAbs, webpAbs := genOptimizedAbsPath(rawImageAbs, config.ExhaustPath, imgFilename, reqURI, extraParams)
convertFilter(rawImageAbs, avifAbs, webpAbs, extraParams, nil)
var availableFiles = []string{rawImageAbs}
for _, v := range goodFormat {
if v == "avif" {
availableFiles = append(availableFiles, avifAbs)
}
if v == "webp" {
availableFiles = append(availableFiles, webpAbs)
}
}
var finalFileName = findSmallestFiles(availableFiles)
var finalFileExtension = path.Ext(finalFileName)
if finalFileExtension == ".webp" {
c.Set("Content-Type", "image/webp")
} else if finalFileExtension == ".avif" {
c.Set("Content-Type", "image/avif")
}
c.Set("X-Compression-Rate", getCompressionRate(rawImageAbs, finalFileName))
return c.SendFile(finalFileName)
}
func proxyHandler(c *fiber.Ctx, reqURIwithQuery string) (string, error) {
// https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200
realRemoteAddr := config.ImgPath + reqURIwithQuery
// Ping Remote for status code and etag info
log.Infof("Remote Addr is %s, fetching info...", realRemoteAddr)
statusCode, etagValue, _ := getRemoteImageInfo(realRemoteAddr)
// Since we cannot store file in format of "/mypic/123.jpg?someother=200&somebugs=200", we need to hash it.
reqURIwithQueryHash := Sha1Path(reqURIwithQuery) // 378e740ca56144b7587f3af9debeee544842879a
etagValueHash := Sha1Path(etagValue) // 123e740ca56333b7587f3af9debeee5448428123
localRawImagePath := path.Join(remoteRaw, reqURIwithQueryHash+"-etag-"+etagValueHash) // For store the remote raw image, /home/webp_server/remote-raw/378e740ca56144b7587f3af9debeee544842879a-etag-123e740ca56333b7587f3af9debeee5448428123
if statusCode == 200 {
if imageExists(localRawImagePath) {
return localRawImagePath, nil
} else {
// Temporary store of remote file.
cleanProxyCache(config.ExhaustPath + reqURIwithQuery + "*")
log.Info("Remote file not found in remote-raw path, fetching...")
err := fetchRemoteImage(localRawImagePath, realRemoteAddr)
return localRawImagePath, err
}
} else {
msg := fmt.Sprintf("Remote returned %d status code!", statusCode)
_ = c.Send([]byte(msg))
log.Warn(msg)
_ = c.SendStatus(statusCode)
cleanProxyCache(config.ExhaustPath + reqURIwithQuery + "*")
return "", errors.New(msg)
}
}

248
router_test.go
View File

@ -1,248 +0,0 @@
package main
import (
"io"
"net/http"
"net/http/httptest"
"os"
"testing"
"time"
"github.com/gofiber/fiber/v2"
"github.com/gofiber/fiber/v2/middleware/etag"
"github.com/patrickmn/go-cache"
"github.com/stretchr/testify/assert"
)
var (
chromeUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36"
acceptWebP = "image/webp,image/apng,image/*,*/*;q=0.8"
acceptAvif = "image/avif,image/*,*/*;q=0.8"
acceptLegacy = "image/jpeg"
safariUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1 Safari/605.1.15"
curlUA = "curl/7.64.1"
)
func TestMain(m *testing.M) {
// setup parameters here...
config.ImgPath = "./pics"
config.ExhaustPath = "./exhaust_test"
config.AllowedTypes = []string{"jpg", "png", "jpeg", "bmp"}
proxyMode = false
remoteRaw = "remote-raw"
WriteLock = cache.New(5*time.Minute, 10*time.Minute)
m.Run()
}
func requestToServer(url string, app *fiber.App, ua, accept string) (*http.Response, []byte) {
headers := make(map[string]string)
headers["User-Agent"] = ua
headers["Accept"] = accept
return requestToServerHeaders(url, app, headers)
}
func requestToServerHeaders(url string, app *fiber.App, headers map[string]string) (*http.Response, []byte) {
req := httptest.NewRequest("GET", url, nil)
for header, value := range headers {
req.Header.Set(header, value)
}
resp, err := app.Test(req, 120000)
if err != nil {
return nil, nil
}
data, _ := io.ReadAll(resp.Body)
return resp, data
}
func TestServerHeaders(t *testing.T) {
var app = fiber.New()
app.Use(etag.New(etag.Config{
Weak: true,
}))
app.Get("/*", convert)
url := "http://127.0.0.1:3333/webp_server.bmp"
// test for chrome
response, _ := requestToServer(url, app, chromeUA, acceptWebP)
defer response.Body.Close()
ratio := response.Header.Get("X-Compression-Rate")
etag := response.Header.Get("Etag")
lastModified := response.Header.Get("Last-Modified")
assert.NotEqual(t, "", ratio)
assert.NotEqual(t, "", etag)
assert.NotEqual(t, "", lastModified)
// TestServerHeadersNotModified
var headers = map[string]string{
"User-Agent": chromeUA,
"Accept": acceptWebP,
"If-None-Match": etag,
}
response, _ = requestToServerHeaders(url, app, headers)
defer response.Body.Close()
assert.Equal(t, 304, response.StatusCode)
headers["If-Modified-Since"] = lastModified
response, _ = requestToServerHeaders(url, app, headers)
defer response.Body.Close()
assert.Equal(t, 304, response.StatusCode)
headers["If-None-Match"] = ""
headers["If-Modified-Since"] = lastModified
response, _ = requestToServerHeaders(url, app, headers)
defer response.Body.Close()
assert.Equal(t, 304, response.StatusCode)
// test for safari
response, _ = requestToServer(url, app, safariUA, acceptLegacy)
defer response.Body.Close()
// ratio = response.Header.Get("X-Compression-Rate")
etag = response.Header.Get("Etag")
lastModified = response.Header.Get("Last-Modified")
assert.NotEqual(t, "", etag)
assert.NotEqual(t, "", lastModified)
}
func TestConvert(t *testing.T) {
// TODO: old-style test, better update it with accept headers
var testChromeLink = map[string]string{
"http://127.0.0.1:3333/webp_server.jpg": "image/webp",
"http://127.0.0.1:3333/webp_server.bmp": "image/webp",
"http://127.0.0.1:3333/webp_server.png": "image/webp",
"http://127.0.0.1:3333/empty.jpg": "",
"http://127.0.0.1:3333/png.jpg": "image/webp",
"http://127.0.0.1:3333/12314.jpg": "",
"http://127.0.0.1:3333/dir1/inside.jpg": "image/webp",
"http://127.0.0.1:3333/%e5%a4%aa%e7%a5%9e%e5%95%a6.png": "image/webp",
"http://127.0.0.1:3333/太神啦.png": "image/webp",
}
var testChromeAvifLink = map[string]string{
"http://127.0.0.1:3333/webp_server.jpg": "image/avif",
"http://127.0.0.1:3333/webp_server.bmp": "image/avif",
"http://127.0.0.1:3333/webp_server.png": "image/avif",
"http://127.0.0.1:3333/empty.jpg": "",
"http://127.0.0.1:3333/png.jpg": "image/avif",
"http://127.0.0.1:3333/12314.jpg": "",
"http://127.0.0.1:3333/dir1/inside.jpg": "image/avif",
"http://127.0.0.1:3333/%e5%a4%aa%e7%a5%9e%e5%95%a6.png": "image/avif",
"http://127.0.0.1:3333/太神啦.png": "image/avif",
}
var testSafariLink = map[string]string{
"http://127.0.0.1:3333/webp_server.jpg": "image/jpeg",
"http://127.0.0.1:3333/webp_server.bmp": "image/bmp",
"http://127.0.0.1:3333/webp_server.png": "image/png",
"http://127.0.0.1:3333/empty.jpg": "",
"http://127.0.0.1:3333/png.jpg": "image/png",
"http://127.0.0.1:3333/12314.jpg": "",
"http://127.0.0.1:3333/dir1/inside.jpg": "image/jpeg",
}
var app = fiber.New()
app.Get("/*", convert)
// test Chrome
for url, respType := range testChromeLink {
resp, data := requestToServer(url, app, chromeUA, acceptWebP)
defer resp.Body.Close()
contentType := getFileContentType(data)
assert.Equal(t, respType, contentType)
}
// test Safari
for url, respType := range testSafariLink {
resp, data := requestToServer(url, app, safariUA, acceptLegacy)
defer resp.Body.Close()
contentType := getFileContentType(data)
assert.Equal(t, respType, contentType)
}
// test Avif is processed in proxy mode
config.EnableAVIF = true
for url, respType := range testChromeAvifLink {
resp, data := requestToServer(url, app, chromeUA, acceptAvif)
defer resp.Body.Close()
contentType := getFileContentType(data)
assert.NotNil(t, respType)
assert.Equal(t, respType, contentType)
}
}
func TestConvertNotAllowed(t *testing.T) {
config.AllowedTypes = []string{"jpg", "png", "jpeg"}
var app = fiber.New()
app.Get("/*", convert)
// not allowed, but we have the file, this should return File extension not allowed
url := "http://127.0.0.1:3333/webp_server.bmp"
resp, data := requestToServer(url, app, chromeUA, acceptWebP)
defer resp.Body.Close()
assert.Contains(t, string(data), "File extension not allowed")
// not allowed, random file
url = url + "hagdgd"
resp, data = requestToServer(url, app, chromeUA, acceptWebP)
defer resp.Body.Close()
assert.Contains(t, string(data), "File extension not allowed")
}
func TestConvertProxyModeBad(t *testing.T) {
proxyMode = true
var app = fiber.New()
app.Get("/*", convert)
// this is local random image, should be 404
url := "http://127.0.0.1:3333/webp_8888server.jpg"
resp, _ := requestToServer(url, app, chromeUA, acceptWebP)
defer resp.Body.Close()
assert.Equal(t, http.StatusNotFound, resp.StatusCode)
// this is local random image, test using cURL, should be 404, ref: https://github.com/webp-sh/webp_server_go/issues/197
resp1, _ := requestToServer(url, app, curlUA, acceptWebP)
defer resp1.Body.Close()
assert.Equal(t, http.StatusNotFound, resp1.StatusCode)
}
func TestConvertProxyModeWork(t *testing.T) {
proxyMode = true
var app = fiber.New()
app.Get("/*", convert)
config.ImgPath = "https://webp.sh"
url := "https://webp.sh/images/cover.jpg"
resp, data := requestToServer(url, app, chromeUA, acceptWebP)
defer resp.Body.Close()
assert.Equal(t, http.StatusOK, resp.StatusCode)
assert.Equal(t, "image/webp", getFileContentType(data))
// test proxyMode with Safari
resp, data = requestToServer(url, app, safariUA, acceptLegacy)
defer resp.Body.Close()
assert.Equal(t, http.StatusOK, resp.StatusCode)
assert.Equal(t, "image/jpeg", getFileContentType(data))
}
func TestConvertBigger(t *testing.T) {
proxyMode = false
config.Quality = 100
config.ImgPath = "./pics"
var app = fiber.New()
app.Get("/*", convert)
url := "http://127.0.0.1:3333/big.jpg"
resp, data := requestToServer(url, app, chromeUA, acceptWebP)
defer resp.Body.Close()
assert.Equal(t, "image/jpeg", resp.Header.Get("content-type"))
assert.Equal(t, "image/jpeg", getFileContentType(data))
_ = os.RemoveAll(config.ExhaustPath)
}

109
webp-server.go
View File

@ -1,77 +1,50 @@
package main
import (
"encoding/json"
"flag"
"fmt"
"os"
"regexp"
"runtime"
"time"
"webp_server_go/config"
"webp_server_go/encoder"
"webp_server_go/handler"
"github.com/davidbyttow/govips/v2/vips"
"github.com/gofiber/fiber/v2"
"github.com/gofiber/fiber/v2/middleware/etag"
"github.com/gofiber/fiber/v2/middleware/logger"
"github.com/patrickmn/go-cache"
log "github.com/sirupsen/logrus"
)
var WriteLock *cache.Cache
var app = fiber.New(fiber.Config{
ServerHeader: "Webp Server Go",
AppName: "Webp Server Go",
DisableStartupMessage: true,
ProxyHeader: "X-Real-IP",
})
func loadConfig(path string) Config {
jsonObject, err := os.Open(path)
if err != nil {
log.Fatal(err)
}
decoder := json.NewDecoder(jsonObject)
_ = decoder.Decode(&config)
_ = jsonObject.Close()
return config
}
func deferInit() {
flag.StringVar(&configPath, "config", "config.json", "/path/to/config.json. (Default: ./config.json)")
flag.BoolVar(&prefetch, "prefetch", false, "Prefetch and convert image to webp")
flag.IntVar(&jobs, "jobs", runtime.NumCPU(), "Prefetch thread, default is all.")
flag.BoolVar(&dumpConfig, "dump-config", false, "Print sample config.json")
flag.BoolVar(&dumpSystemd, "dump-systemd", false, "Print sample systemd service file.")
flag.BoolVar(&verboseMode, "v", false, "Verbose, print out debug info.")
flag.BoolVar(&showVersion, "V", false, "Show version information.")
flag.Parse()
// Logrus
func setupLogger() {
log.SetOutput(os.Stdout)
log.SetReportCaller(true)
Formatter := &log.TextFormatter{
formatter := &log.TextFormatter{
EnvironmentOverrideColors: true,
FullTimestamp: true,
TimestampFormat: "2006-01-02 15:04:05",
TimestampFormat: config.TimeDateFormat,
CallerPrettyfier: func(f *runtime.Frame) (string, string) {
return fmt.Sprintf("[%d:%s()]", f.Line, f.Function), ""
return fmt.Sprintf("[%d:%s]", f.Line, f.Function), ""
},
}
log.SetFormatter(Formatter)
log.SetFormatter(formatter)
log.SetLevel(log.InfoLevel)
if verboseMode {
log.SetLevel(log.DebugLevel)
log.Debug("Debug mode is enabled!")
} else {
log.SetLevel(log.InfoLevel)
}
// fiber logger format
app.Use(logger.New(logger.Config{
Format: config.FiberLogFormat,
TimeFormat: config.TimeDateFormat,
}))
log.Infoln("Logger ready.")
}
func switchProxyMode() {
// Check for remote address
matched, _ := regexp.MatchString(`^https?://`, config.ImgPath)
proxyMode = false
if matched {
proxyMode = true
} else {
_, err := os.Stat(config.ImgPath)
if err != nil {
log.Fatalf("Your image path %s is incorrect.Please check and confirm.", config.ImgPath)
}
}
func init() {
setupLogger()
}
func main() {
@ -83,48 +56,32 @@ func main() {
▘ ▘▝▀▘▀▀ ▘ ▝▀ ▝▀▘▘ ▘ ▝▀▘▘ ▝▀ ▝▀
Webp Server Go - v%s
Develop by WebP Server team. https://github.com/webp-sh`, version)
Develop by WebP Server team. https://github.com/webp-sh`, config.Version)
deferInit()
// process cli params
if dumpConfig {
fmt.Println(sampleConfig)
if config.DumpConfig {
fmt.Println(config.SampleConfig)
os.Exit(0)
}
if dumpSystemd {
fmt.Println(sampleSystemd)
if config.DumpSystemd {
fmt.Println(config.SampleSystemd)
os.Exit(0)
}
if showVersion {
if config.ShowVersion {
fmt.Printf("\n %c[1;32m%s%c[0m\n\n", 0x1B, banner+"", 0x1B)
os.Exit(0)
}
config = loadConfig(configPath)
switchProxyMode()
vips.Startup(&vips.Config{
ConcurrencyLevel: runtime.NumCPU(),
})
defer vips.Shutdown()
WriteLock = cache.New(5*time.Minute, 10*time.Minute)
if prefetch {
go prefetchImages(config.ImgPath, config.ExhaustPath)
if config.Prefetch {
go encoder.PrefetchImages()
}
app := fiber.New(fiber.Config{
ServerHeader: "Webp Server Go",
DisableStartupMessage: true,
})
app.Use(etag.New(etag.Config{
Weak: true,
}))
app.Use(logger.New())
listenAddress := config.Host + ":" + config.Port
app.Get("/*", convert)
listenAddress := config.Config.Host + ":" + config.Config.Port
app.Get("/*", handler.Convert)
fmt.Printf("\n %c[1;32m%s%c[0m\n\n", 0x1B, banner, 0x1B)
fmt.Println("Webp Server Go is Running on http://" + listenAddress)

72
webp-server_test.go
View File

@ -1,72 +0,0 @@
package main
import (
"net"
"os"
"runtime"
"testing"
"time"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
)
// due to test limit, we can't test for cli param part.
func TestLoadConfig(t *testing.T) {
c := loadConfig("./config.json")
assert.Equal(t, "./exhaust", c.ExhaustPath)
assert.Equal(t, "127.0.0.1", c.Host)
assert.Equal(t, "3333", c.Port)
assert.Equal(t, 80, c.Quality)
assert.Equal(t, "./pics", c.ImgPath)
assert.Equal(t, []string{"jpg", "png", "jpeg", "bmp", "gif"}, c.AllowedTypes)
}
func TestDeferInit(t *testing.T) {
// test initial value
assert.Equal(t, "", configPath)
assert.False(t, prefetch)
assert.Equal(t, false, dumpSystemd)
assert.Equal(t, false, dumpConfig)
assert.False(t, verboseMode)
}
func TestMainFunction(t *testing.T) {
// first test verbose mode
assert.False(t, verboseMode)
assert.Equal(t, log.GetLevel(), log.InfoLevel)
os.Args = append(os.Args, "-v", "-prefetch")
// run main function
go main()
time.Sleep(time.Second * 5)
// verbose, prefetch
assert.Equal(t, log.GetLevel(), log.DebugLevel)
assert.True(t, verboseMode)
assert.True(t, prefetch)
// test read config value
assert.Equal(t, "config.json", configPath)
assert.Equal(t, runtime.NumCPU(), jobs)
assert.Equal(t, false, dumpSystemd)
assert.Equal(t, false, dumpConfig)
// test port
conn, err := net.DialTimeout("tcp", net.JoinHostPort("127.0.0.1", "3333"), time.Second*2)
assert.Nil(t, err)
assert.NotNil(t, conn)
}
func TestProxySwitch(t *testing.T) {
// real proxy mode
assert.False(t, proxyMode)
config.ImgPath = "https://z.cn"
switchProxyMode()
assert.True(t, proxyMode)
// normal
config.ImgPath = os.TempDir()
switchProxyMode()
assert.False(t, proxyMode)
}