wood/webp_server_go
1
0
Fork 0
mirror of https://github.com/woodchen-ink/webp_server_go.git synced 2025-07-18 13:42:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #81 from webp-sh/avif

Browse Source
This commit is contained in:
Benny 2021-12-12 14:17:09 +08:00 committed by GitHub
commit 8b0354fff7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 485 additions and 387 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.dockerignore Normal file
View File

@ -0,0 +1,9 @@
.github/*
exhaust/*
pics/*
remote-raw/*
scripts/*
.git/*
README.md
LICENSE
Makefile

2
.github/dependabot.yml vendored
View File

@ -1,6 +1,6 @@
version: 2
updates:
- package-ecosystem: ""
- package-ecosystem: "gomod"
directory: "/"
schedule:
interval: "daily"

21
.github/workflows/CI.yaml vendored Normal file
View File

@ -0,0 +1,21 @@
name: CI
on:
push:
jobs:
ci:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
with:
submodules: true
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.17.3
- name: run test cases
run: sudo apt install libaom-dev && make test && make

10
Dockerfile
View File

@ -1,16 +1,16 @@
FROM golang:alpine as builder
FROM golang:1.17.4-alpine as builder
ARG IMG_PATH=/opt/pics
ARG EXHAUST_PATH=/opt/exhaust
RUN apk update && apk add alpine-sdk && mkdir /build
RUN apk update && apk add alpine-sdk aom-dev && mkdir /build
COPY go.mod /build
RUN cd /build && go mod download
COPY . /build
RUN cd /build && sed -i "s|.\/pics|${IMG_PATH}|g" config.json \
&& sed -i "s|\"\"|\"${EXHAUST_PATH}\"|g" config.json \
&& sed -i 's/127.0.0.1/0.0.0.0/g' config.json \
&& make docker
&& sed -i "s|\"\"|\"${EXHAUST_PATH}\"|g" config.json \
&& sed -i 's/127.0.0.1/0.0.0.0/g' config.json \
&& go build -ldflags="-s -w" -o webp-server .

3
Makefile
View File

@ -14,6 +14,7 @@ default:
make clean
go build -o builds/webp-server-$(OS)-$(ARCH) .
ls builds
all:
make clean
./scripts/build.sh $(OS) $(ARCH)
@ -26,4 +27,4 @@ clean:
rm -rf prefetch
docker:
go build -ldflags="-s -w" -o webp-server .
DOCKER_BUILDKIT=1 docker build -t webpsh/webps .

13
config.go
View File

@ -8,9 +8,10 @@ type Config struct {
Host string `json:"HOST"`
Port string `json:"PORT"`
ImgPath string `json:"IMG_PATH"`
Quality string `json:"QUALITY"`
Quality float32 `json:"QUALITY,string"`
AllowedTypes []string `json:"ALLOWED_TYPES"`
ExhaustPath string `json:"EXHAUST_PATH"`
EnableAVIF bool `json:"ENABLE_AVIF"`
}
var (
@ -21,7 +22,7 @@ var (
prefetch, proxyMode bool
remoteRaw = "remote-raw"
config Config
version = "0.3.2"
version = "0.4.0"
releaseUrl = "https://github.com/webp-sh/webp_server_go/releases/latest/download/"
)
@ -33,7 +34,8 @@ const (
"QUALITY": "80",
"IMG_PATH": "./pics",
"EXHAUST_PATH": "./exhaust",
"ALLOWED_TYPES": ["jpg","png","jpeg","bmp"]
"ALLOWED_TYPES": ["jpg","png","jpeg","bmp"],
"ENABLE_AVIF": false
}`
sampleSystemd = `
@ -53,3 +55,8 @@ RestartSec=3s
[Install]
WantedBy=multi-user.target`
)
const (
webpMax = 16383
avifMax = 65536
)

172
encoder.go
View File

@ -3,70 +3,152 @@ package main
import (
"bytes"
"errors"
"fmt"
"github.com/Kagami/go-avif"
"github.com/chai2010/webp"
log "github.com/sirupsen/logrus"
"golang.org/x/image/bmp"
"image"
"image/gif"
"image/jpeg"
"image/png"
"io/ioutil"
"os"
"path"
"path/filepath"
"runtime"
"strings"
log "github.com/sirupsen/logrus"
"github.com/chai2010/webp"
"golang.org/x/image/bmp"
)
func webpEncoder(p1, p2 string, quality float32, Log bool, c chan int) (err error) {
// if convert fails, return error; success nil
func convertFilter(raw, avifPath, webpPath string, c chan int) {
// all absolute paths
log.Debugf("target: %s with quality of %f", path.Base(p1), quality)
var buf bytes.Buffer
var img image.Image
if !imageExists(avifPath) && config.EnableAVIF {
convertImage(raw, avifPath, "avif")
}
data, err := ioutil.ReadFile(p1)
if !imageExists(webpPath) {
convertImage(raw, webpPath, "webp")
}
if c != nil {
c <- 1
}
}
func convertImage(raw, optimized, itype string) {
// we don't have abc.jpg.png1582558990.webp
// delete the old pic and convert a new one.
// optimized: /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
// we'll delete file starts with /home/webp_server/exhaust/path/to/tsuki.jpg.ts.itype
s := strings.Split(path.Base(optimized), ".")
pattern := path.Join(path.Dir(optimized), s[0]+"."+s[1]+".*."+s[len(s)-1])
matches, err := filepath.Glob(pattern)
if err != nil {
chanErr(c)
return
log.Error(err.Error())
} else {
for _, p := range matches {
_ = os.Remove(p)
}
}
//we need to create dir first
err = os.MkdirAll(path.Dir(optimized), 0755)
//q, _ := strconv.ParseFloat(config.Quality, 32)
switch itype {
case "webp":
webpEncoder(raw, optimized, config.Quality)
case "avif":
avifEncoder(raw, optimized, config.Quality)
}
}
func readRawImage(imgPath string, maxPixel int) (img image.Image, err error) {
data, err := ioutil.ReadFile(imgPath)
if err != nil {
log.Errorln(err)
}
contentType := getFileContentType(data[:512])
if strings.Contains(contentType, "jpeg") {
img, _ = jpeg.Decode(bytes.NewReader(data))
img, err = jpeg.Decode(bytes.NewReader(data))
} else if strings.Contains(contentType, "png") {
img, _ = png.Decode(bytes.NewReader(data))
img, err = png.Decode(bytes.NewReader(data))
} else if strings.Contains(contentType, "bmp") {
img, _ = bmp.Decode(bytes.NewReader(data))
} else if strings.Contains(contentType, "gif") {
// TODO: need to support animated webp
log.Warn("Gif support is not perfect!")
img, _ = gif.Decode(bytes.NewReader(data))
img, err = bmp.Decode(bytes.NewReader(data))
}
if err != nil || img == nil {
errinfo := fmt.Sprintf("image file %s is corrupted: %v", imgPath, err)
log.Errorln(errinfo)
return nil, errors.New(errinfo)
}
if img == nil {
msg := "image file " + path.Base(p1) + " is corrupted or not supported"
log.Debug(msg)
err = errors.New(msg)
chanErr(c)
return
x, y := img.Bounds().Max.X, img.Bounds().Max.Y
if x > maxPixel || y > maxPixel {
errinfo := fmt.Sprintf("WebP: %s(%dx%d) is too large", imgPath, x, y)
log.Warnf(errinfo)
return nil, errors.New(errinfo)
}
if err = webp.Encode(&buf, img, &webp.Options{Lossless: false, Quality: quality}); err != nil {
log.Error(err)
chanErr(c)
return
}
if err = ioutil.WriteFile(p2, buf.Bytes(), 0644); err != nil {
log.Error(err)
chanErr(c)
return
}
if Log {
log.Info("Save to " + p2 + " ok!\n")
}
chanErr(c)
return nil
return img, nil
}
func avifEncoder(p1, p2 string, quality float32) {
var img image.Image
dst, err := os.Create(p2)
if err != nil {
log.Fatalf("Can't create destination file: %v", err)
}
// AVIF has a maximum resolution of 65536 x 65536 pixels.
img, err = readRawImage(p1, avifMax)
if err != nil {
return
}
err = avif.Encode(dst, img, &avif.Options{
Threads: runtime.NumCPU(),
Speed: avif.MaxSpeed,
Quality: int((100 - quality) / 100 * avif.MaxQuality),
SubsampleRatio: nil,
})
if err != nil {
log.Warnf("Can't encode source image: %v to AVIF", err)
}
convertLog("AVIF", p1, p2, quality)
}
func webpEncoder(p1, p2 string, quality float32) {
// if convert fails, return error; success nil
var buf bytes.Buffer
var img image.Image
// The maximum pixel dimensions of a WebP image is 16383 x 16383.
img, err := readRawImage(p1, webpMax)
if err != nil {
return
}
err = webp.Encode(&buf, img, &webp.Options{Lossless: false, Quality: quality})
if err != nil {
log.Warnf("Can't encode source image: %v to WebP", err)
}
if err := ioutil.WriteFile(p2, buf.Bytes(), 0644); err != nil {
log.Error(err)
return
}
convertLog("WebP", p1, p2, quality)
}
func convertLog(itype, p1 string, p2 string, quality float32) {
oldf, _ := os.Stat(p1)
newf, _ := os.Stat(p2)
log.Infof("%s@%.2f%%: %s->%s %d->%d %.2f%% deflated", itype, quality,
p1, p2, oldf.Size(), newf.Size(), float32(newf.Size())/float32(oldf.Size())*100)
}

66
encoder_test.go
View File

@ -4,65 +4,63 @@ import (
"github.com/stretchr/testify/assert"
"io/ioutil"
"os"
"path"
"path/filepath"
"strings"
"testing"
)
func walker() []string {
var list []string
_ = filepath.Walk("./pics", func(path string, info os.FileInfo, err error) error {
if !info.IsDir() {
list = append(list, path)
_ = filepath.Walk("./pics", func(p string, info os.FileInfo, err error) error {
if !info.IsDir() && !strings.HasPrefix(path.Base(p), ".") {
list = append(list, p)
}
return nil
})
return list
}
func TestWebpEncoder(t *testing.T) {
var webp = "/tmp/test-result.webp"
func TestWebPEncoder(t *testing.T) {
// Go through every files
var dest = "/tmp/test-result"
var target = walker()
for _, f := range target {
//fmt.Println(b, c, webp)
runEncoder(t, f, webp)
runEncoder(t, f, dest)
}
_ = os.Remove(webp)
_ = os.Remove(dest)
}
// test error
err := webpEncoder("./pics/empty.jpg", webp, 80, true, nil)
assert.NotNil(t, err)
_ = os.Remove(webp)
func TestAvifEncoder(t *testing.T) {
// Only one file: img_over_16383px.jpg might cause memory issues on CI environment
var dest = "/tmp/test-result"
avifEncoder("./pics/big.jpg", dest, 80)
assertType(t, dest, "image/avif")
}
func TestNonExistImage(t *testing.T) {
var webp = "/tmp/test-result.webp"
// test error
var err = webpEncoder("./pics/empty.jpg", webp, 80, true, nil)
assert.NotNil(t, err)
_ = os.Remove(webp)
var dest = "/tmp/test-result"
webpEncoder("./pics/empty.jpg", dest, 80)
avifEncoder("./pics/empty.jpg", dest, 80)
}
func TestConvertFail(t *testing.T) {
var webp = "/tmp/test-result.webp"
var err = webpEncoder("./pics/webp_server.jpg", webp, -1, true, nil)
assert.NotNil(t, t, err)
var dest = "/tmp/test-result"
webpEncoder("./pics/webp_server.jpg", dest, -1)
avifEncoder("./pics/webp_server.jpg", dest, -1)
}
func runEncoder(t *testing.T, file string, webp string) {
var c chan int
//t.Logf("convert from %s to %s", file, webp)
var err = webpEncoder(file, webp, 80, true, c)
if file == "pics/empty.jpg" && err != nil {
func runEncoder(t *testing.T, file string, dest string) {
if file == "pics/empty.jpg" {
t.Log("Empty file, that's okay.")
} else if err != nil {
t.Fatalf("Fatal, convert failed for %s: %v ", file, err)
}
data, _ := ioutil.ReadFile(webp)
types := getFileContentType(data[:512])
if types != "image/webp" {
t.Fatal("Fatal, file type is wrong!")
}
webpEncoder(file, dest, 80)
assertType(t, dest, "image/webp")
}
func assertType(t *testing.T, dest, mime string) {
data, _ := ioutil.ReadFile(dest)
types := getFileContentType(data[:512])
assert.Equalf(t, mime, types, "File %s should be %s", dest, mime)
}

9
go.mod
View File

@ -3,11 +3,18 @@ module webp_server_go
go 1.15
require (
github.com/Kagami/go-avif v0.1.0
github.com/chai2010/webp v1.1.0
github.com/gofiber/fiber/v2 v2.4.0
github.com/sirupsen/logrus v1.6.0
github.com/h2non/filetype v1.1.3
github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
github.com/schollz/progressbar/v3 v3.8.3
github.com/sirupsen/logrus v1.8.1
github.com/staktrace/go-update v0.0.0-20210525161054-fc019945f9a2
github.com/stretchr/testify v1.3.0
github.com/valyala/fasthttp v1.18.0
golang.org/x/image v0.0.0-20200119044424-58c23975cae1
golang.org/x/sys v0.0.0-20211204120058-94396e421777 // indirect
)
replace (

37
go.sum
View File

@ -1,15 +1,37 @@
github.com/Kagami/go-avif v0.1.0 h1:8GHAGLxCdFfhpd4Zg8j1EqO7rtcQNenxIDerC/uu68w=
github.com/Kagami/go-avif v0.1.0/go.mod h1:OPmPqzNdQq3+sXm0HqaUJQ9W/4k+Elbc3RSfJUemDKA=
github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4=
github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
github.com/klauspost/compress v1.10.7 h1:7rix8v8GpI3ZBb0nSozFRgbtXKv+hOe+qfEpZqybrAg=
github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/schollz/progressbar/v3 v3.8.3 h1:FnLGl3ewlDUP+YdSwveXBaXs053Mem/du+wr7XSYKl8=
github.com/schollz/progressbar/v3 v3.8.3/go.mod h1:pWnVCjSBZsT2X3nx9HfRdnCDrpbevliMeoEVhStwHko=
github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/staktrace/go-update v0.0.0-20210525161054-fc019945f9a2 h1:kyTDvRL8TyTHOx0aK1PKMPVfkI7rCLDC1nrKF7SYOBc=
github.com/staktrace/go-update v0.0.0-20210525161054-fc019945f9a2/go.mod h1:yzHsbjWRHVF1vVGsluwqCc2TvogFSx9C8TeBh34cMb0=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
@ -23,16 +45,29 @@ github.com/webp-sh/webp v1.2.0 h1:WiAR1M4Cz50Ehv0vP8VaBZwuTqBgLxcGMaBV23zLVDA=
github.com/webp-sh/webp v1.2.0/go.mod h1:DWmBXPtpA/zfTgEgWxAlsER3B7nXFvDtCi1YY+K5N9w=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ=
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/image v0.0.0-20200119044424-58c23975cae1 h1:5h3ngYt7+vXCDZCup/HkCQgW5XwmSvR/nA2JmJ0RErg=
golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201210223839-7e3030f88018 h1:XKi8B/gRBuTZN1vU9gFsLMm6zVz5FSCDzm8JYACnjy8=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201210223839-7e3030f88018/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0 h1:xrCZDmdtoloIiooiA9q0OQb9r8HejIHYoHGhGCe1pGg=
golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211204120058-94396e421777 h1:QAkhGVjOxMa+n4mlsAWeAU+BMZmimQAaNiMu+iUi94E=
golang.org/x/sys v0.0.0-20211204120058-94396e421777/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE=
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

159
helper.go
View File

@ -1,6 +1,7 @@
package main
import (
"bytes"
"fmt"
"hash/crc32"
"io"
@ -11,26 +12,35 @@ import (
"path/filepath"
"strconv"
"github.com/valyala/fasthttp"
"strings"
"github.com/h2non/filetype"
log "github.com/sirupsen/logrus"
)
func chanErr(ccc chan int) {
if ccc != nil {
ccc <- 1
}
func avifMatcher(buf []byte) bool {
// 0000001C 66747970 6D696631 00000000 6D696631 61766966 6D696166 000000F4
return len(buf) > 1 && bytes.Equal(buf[:28], []byte{
0x0, 0x0, 0x0, 0x1c,
0x66, 0x74, 0x79, 0x70,
0x6d, 0x69, 0x66, 0x31,
0x0, 0x0, 0x0, 0x0,
0x6d, 0x69, 0x66, 0x31,
0x61, 0x76, 0x69, 0x66,
0x6d, 0x69, 0x61, 0x66,
})
}
func getFileContentType(buffer []byte) string {
// Use the net/http package's handy DectectContentType function. Always returns a valid
// content-type by returning "application/octet-stream" if no others seemed to match.
contentType := http.DetectContentType(buffer)
return contentType
var avifType = filetype.NewType("avif", "image/avif")
filetype.AddMatcher(avifType, avifMatcher)
kind, _ := filetype.Match(buffer)
return kind.MIME.Value
}
func fileCount(dir string) int {
count := 0
func fileCount(dir string) int64 {
var count int64 = 0
_ = filepath.Walk(dir,
func(path string, info os.FileInfo, err error) error {
if err != nil {
@ -53,6 +63,17 @@ func imageExists(filename string) bool {
return !info.IsDir()
}
func checkAllowedType(imgFilename string) bool {
imgFilename = strings.ToLower(imgFilename)
for _, allowedType := range config.AllowedTypes {
allowedType = "." + strings.ToLower(allowedType)
if strings.HasSuffix(imgFilename, allowedType) {
return true
}
}
return false
}
// Check for remote filepath, e.g: https://test.webp.sh/node.png
// return StatusCode, etagValue and length
func getRemoteImageInfo(fileUrl string) (int, string, string) {
@ -106,22 +127,24 @@ func cleanProxyCache(cacheImagePath string) {
}
}
func genWebpAbs(RawImagePath string, ExhaustPath string, ImgFilename string, reqURI string) (string, string) {
func genOptimizedAbs(rawImagePath string, exhaustPath string, imageName string, reqURI string) (string, string) {
// get file mod time
STAT, err := os.Stat(RawImagePath)
STAT, err := os.Stat(rawImagePath)
if err != nil {
log.Error(err.Error())
return "", ""
}
ModifiedTime := STAT.ModTime().Unix()
// webpFilename: abc.jpg.png -> abc.jpg.png.1582558990.webp
WebpFilename := fmt.Sprintf("%s.%d.webp", ImgFilename, ModifiedTime)
cwd, _ := os.Getwd()
webpFilename := fmt.Sprintf("%s.%d.webp", imageName, ModifiedTime)
// avifFilename: abc.jpg.png -> abc.jpg.png.1582558990.avif
avifFilename := fmt.Sprintf("%s.%d.avif", imageName, ModifiedTime)
// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
// Custom Exhaust: /path/to/exhaust/web_path/web_to/tsuki.jpg.1582558990.webp
WebpAbsolutePath := path.Clean(path.Join(ExhaustPath, path.Dir(reqURI), WebpFilename))
return cwd, WebpAbsolutePath
webpAbsolutePath := path.Clean(path.Join(exhaustPath, path.Dir(reqURI), webpFilename))
avifAbsolutePath := path.Clean(path.Join(exhaustPath, path.Dir(reqURI), avifFilename))
return avifAbsolutePath, webpAbsolutePath
}
func genEtag(ImgAbsPath string) string {
@ -136,64 +159,60 @@ func genEtag(ImgAbsPath string) string {
return fmt.Sprintf(`W/"%d-%08X"`, len(data), crc)
}
func getCompressionRate(RawImagePath string, webpAbsPath string) string {
func getCompressionRate(RawImagePath string, optimizedImg string) string {
originFileInfo, err := os.Stat(RawImagePath)
if err != nil {
log.Warnf("fail to get raw image %v", err)
log.Warnf("Failed to get raw image %v", err)
return ""
}
webpFileInfo, err := os.Stat(webpAbsPath)
optimizedFileInfo, err := os.Stat(optimizedImg)
if err != nil {
log.Warnf("fail to get webp image %v", err)
log.Warnf("Failed to get optimized image %v", err)
return ""
}
compressionRate := float64(webpFileInfo.Size()) / float64(originFileInfo.Size())
log.Debugf("The compress rate is %d/%d=%.2f", originFileInfo.Size(), webpFileInfo.Size(), compressionRate)
compressionRate := float64(optimizedFileInfo.Size()) / float64(originFileInfo.Size())
log.Debugf("The compression rate is %d/%d=%.2f", originFileInfo.Size(), optimizedFileInfo.Size(), compressionRate)
return fmt.Sprintf(`%.2f`, compressionRate)
}
func goOrigin(header, ua string) bool {
// We'll first check accept headers, if accept headers is false, we'll then go to UA part
if headerOrigin(header) && uaOrigin(ua) {
return true
} else {
return false
}
}
func guessSupportedFormat(header *fasthttp.RequestHeader) []string {
var supported = map[string]bool{
"raw": true,
"webp": false,
"avif": false}
func uaOrigin(ua string) bool {
// iOS 14 and iPadOS 14 supports webp, the identification token is iPhone OS 14_2_1 and CPU OS 14_2
// for more information, please check test case
if strings.Contains(ua, "iPhone OS 14") || strings.Contains(ua, "CPU OS 14") {
// this is iOS 14/iPadOS 14
return false
} else if strings.Contains(ua, "Firefox") || strings.Contains(ua, "Chrome") {
// Chrome or firefox on macOS Windows
var ua = string(header.Peek("user-agent"))
var accept = strings.ToLower(string(header.Peek("accept")))
log.Debugf("%s\t%s\n", ua, accept)
if strings.Contains(accept, "image/webp") {
supported["webp"] = true
}
if strings.Contains(accept, "image/avif") {
supported["avif"] = true
}
// chrome on iOS will not send valid image accept header
if strings.Contains(ua, "iPhone OS 14") || strings.Contains(ua, "CPU OS 14") ||
strings.Contains(ua, "iPhone OS 15") || strings.Contains(ua, "CPU OS 15") {
supported["webp"] = true
} else if strings.Contains(ua, "Android") || strings.Contains(ua, "Linux") {
// on Android and Linux
} else if strings.Contains(ua, "FxiOS") || strings.Contains(ua, "CriOS") {
//firefox and Chrome on iOS
return true
} else {
return true
supported["webp"] = true
}
return false
var accepted []string
for k, v := range supported {
if v {
accepted = append(accepted, k)
}
}
return accepted
}
func headerOrigin(header string) bool {
// Webkit is really weird especially on iOS, it doesn't even send out effective accept headers.
// Head to test case if you want to know more
if strings.Contains(header, "image/webp") {
return false
} else {
// go to origin
return true
}
}
func chooseProxy(proxyRawSize string, webpAbsPath string) bool {
func chooseProxy(proxyRawSize string, optimizedAbs string) bool {
var proxyRaw, _ = strconv.Atoi(proxyRawSize)
webp, _ := ioutil.ReadFile(webpAbsPath)
webp, _ := ioutil.ReadFile(optimizedAbs)
if len(webp) > proxyRaw {
return true
} else {
@ -201,12 +220,20 @@ func chooseProxy(proxyRawSize string, webpAbsPath string) bool {
}
}
func chooseLocalSmallerFile(rawImageAbs, webpAbsPath string) string {
raw, _ := ioutil.ReadFile(rawImageAbs)
webp, _ := ioutil.ReadFile(webpAbsPath)
if len(webp) > len(raw) {
return rawImageAbs
} else {
return webpAbsPath
func findSmallestFiles(files []string) string {
// walk files
var small int64
var final string
for _, f := range files {
stat, err := os.Stat(f)
if err != nil {
log.Warnf("%s not found on filesystem", f)
continue
}
if stat.Size() < small || small == 0 {
small = stat.Size()
final = f
}
}
return final
}

155
helper_test.go
View File

@ -1,31 +1,33 @@
package main
import (
log "github.com/sirupsen/logrus"
"github.com/valyala/fasthttp"
"io/ioutil"
"net/http"
"path/filepath"
"sort"
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
// test this file: go test -v -cover .
// test all files: go test -v -cover .
// test one case: go test -v -run TestSelectFormat
func TestGetFileContentType(t *testing.T) {
var data = []byte("hello")
var expected = "text/plain; charset=utf-8"
var data = []byte("remember remember the 5th of november")
var expected = ""
var result = getFileContentType(data)
assert.Equalf(t, result, expected, "Result: [%s], Expected: [%s]", result, expected)
}
func TestFileCount(t *testing.T) {
var data = ".github"
var expected = 2
var data = "scripts"
var expected int64 = 2
var result = fileCount(data)
assert.Equalf(t, result, expected, "Result: [%d], Expected: [%d]", result, expected)
}
func TestImageExists(t *testing.T) {
@ -43,7 +45,7 @@ func TestImageExists(t *testing.T) {
}
func TestGenWebpAbs(t *testing.T) {
cwd, cooked := genWebpAbs("./pics/webp_server.png", "/tmp",
cwd, cooked := genOptimizedAbs("./pics/webp_server.png", "/tmp",
"test", "a")
if !strings.Contains(cwd, "webp_server_go") {
t.Logf("Result: [%v], Expected: [%v]", cwd, "webp_server_go")
@ -71,108 +73,63 @@ func TestGenEtag(t *testing.T) {
}
func TestGoOrigin(t *testing.T) {
func TestSelectFormat(t *testing.T) {
// this is a complete test case for webp compatibility
// func goOrigin(header, ua string) bool
// UNLESS YOU KNOW WHAT YOU ARE DOING, DO NOT CHANGE THE TEST CASE MAPPING HERE.
var testCase = map[[2]string]bool{
var fullSupport = []string{"avif", "webp", "raw"}
var webpSupport = []string{"webp", "raw"}
var jpegSupport = []string{"raw"}
var testCase = map[[2]string][]string{
// Latest Chrome on Windows, macOS, linux, Android and iOS 13
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36"}: fullSupport,
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36"}: fullSupport,
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36"}: fullSupport,
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", "Mozilla/5.0 (Linux; Android 10) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.60 Mobile Safari/537.36"}: fullSupport,
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", "Mozilla/5.0 (Linux; Android 6.0; HTC M8t) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.74 Mobile Safari/537.36"}: fullSupport,
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", "HTCM8t_LTE/1.0 Android/4.4 release/2013 Browser/WAP2.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 AppleWebKit/534.30"}: webpSupport,
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 13_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/83.0.4103.63 Mobile/15E148 Safari/604.1"}: jpegSupport,
// macOS Catalina Safari
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.1 Safari/605.1.15"}: true,
// macOS Chrome
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.88 Safari/537.36"}: false,
// iOS14 Safari
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 14_2_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.1 Mobile/15E148 Safari/604.1"}: false,
// iOS14 Chrome
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 14_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/87.0.4280.77 Mobile/15E148 Safari/604.1"}: false,
// iPadOS 14 Safari
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPad; CPU OS 14_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.1 Mobile/15E148 Safari/604.1"}: false,
// iPadOS 14 Chrome
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPad; CPU OS 14_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/87.0.4280.77 Mobile/15E148 Safari/604.1"}: false,
// Warning: these three are real capture headers - I don't have iOS/iPadOS prior to version 14
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1 Safari/605.1.15"}: true,
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPad; CPU OS 10_15_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/25.0 Mobile/15E148 Safari/605.1.15"}: true,
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 13_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/83.0.4103.63 Mobile/15E148 Safari/604.1"}: true,
}
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.1 Safari/605.1.15"}: jpegSupport,
for value, is := range testCase {
assert.Equalf(t, is, goOrigin(value[0], value[1]), "[%v]:[%s]", value, is)
}
}
// iOS14 Safari and Chrome
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 14_2_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.1 Mobile/15E148 Safari/604.1"}: webpSupport,
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 14_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/87.0.4280.77 Mobile/15E148 Safari/604.1"}: webpSupport,
func TestUaOrigin(t *testing.T) {
// reference: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent/Firefox
// https://developer.chrome.com/multidevice/user-agent#chrome_for_ios_user_agent
// iPadOS 14 Safari and Chrome
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPad; CPU OS 14_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.1 Mobile/15E148 Safari/604.1"}: webpSupport,
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPad; CPU OS 14_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/87.0.4280.77 Mobile/15E148 Safari/604.1"}: webpSupport,
// UNLESS YOU KNOW WHAT YOU ARE DOING, DO NOT CHANGE THE TEST CASE MAPPING HERE.
var testCase = map[string]bool{
// Chrome on Windows, macOS, linux, iOS and Android
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36": false,
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36": false,
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36": false,
"Mozilla/5.0 (iPhone; CPU iPhone OS 13_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/83.0.4103.63 Mobile/15E148 Safari/604.1": true,
"Mozilla/5.0 (Linux; Android 10) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.60 Mobile Safari/537.36": false,
// Firefox on Windows, macOS, linux, iOS and Android
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:76.0) Gecko/20100101 Firefox/76.0": false,
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:76.0) Gecko/20100101 Firefox/76.0": false,
"Mozilla/5.0 (X11; Linux i686; rv:76.0) Gecko/20100101 Firefox/76.0": false,
"Mozilla/5.0 (iPad; CPU OS 10_15_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/25.0 Mobile/15E148 Safari/605.1.15": true,
"Mozilla/5.0 (Android 10; Mobile; rv:68.0) Gecko/68.0 Firefox/68.0": false,
// Safari on macOS and iOS
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1 Safari/605.1.15": true,
"Mozilla/5.0 (iPad; CPU OS 13_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Mobile/15E148 Safari/604.1": true,
// WeChat on iOS, Windows, and Android
"Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_3 like Mac OS X) AppleWebKit/603.3.8 (KHTML, like Gecko) Mobile/14G60 wxwork/2.1.5 MicroMessenger/6.3.22": true,
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 MicroMessenger/6.5.2.501 NetType/WIFI WindowsWechat QBCore/3.43.691.400 QQBrowser/9.0.2524.400": false,
"Mozilla/5.0 (Linux; Android 7.0; LG-H831 Build/NRD90U; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/68.0.3440.91 Mobile Safari/537.36 MicroMessenger/6.6.7.1303(0x26060743) NetType/WIFI Language/zh_TW": false,
// iOS 15 Safari, Firefox and Chrome
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 15_1_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Mobile/15E148 Safari/604.1"}: webpSupport,
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 15_1_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/39.0 Mobile/15E148 Safari/605.1.15"}: webpSupport,
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 15_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/96.0.4664.53 Mobile/15E148 Safari/604.1"}: webpSupport,
// IE
"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko": true,
"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)": true,
[2]string{"application/x-ms-application, image/jpeg, application/xaml+xml, image/gif, image/pjpeg, application/x-ms-xbap, */*", "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko"}: jpegSupport,
// Others
"PostmanRuntime/7.26.1": true,
"curl/7.64.1": true,
[2]string{"", "PostmanRuntime/7.26.1"}: jpegSupport,
[2]string{"", "curl/7.64.1"}: jpegSupport,
[2]string{"image/webp", "curl/7.64.1"}: webpSupport,
[2]string{"image/avif,image/webp", "curl/7.64.1"}: fullSupport,
// these four are captured from iOS14/iPadOS14, which supports webp
"Mozilla/5.0 (iPhone; CPU iPhone OS 14_2_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.1 Mobile/15E148 Safari/604.1": false,
"Mozilla/5.0 (iPhone; CPU iPhone OS 14_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/87.0.4280.77 Mobile/15E148 Safari/604.1\n": false,
"Mozilla/5.0 (iPad; CPU OS 14_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.1 Mobile/15E148 Safari/604.1": false,
"Mozilla/5.0 (iPad; CPU OS 14_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/87.0.4280.77 Mobile/15E148 Safari/604.1": false,
// some weird browsers
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 15_1_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 MicroMessenger/8.0.16(0x18001033) NetType/WIFI Language/zh_CN"}: webpSupport,
[2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", "Mozilla/5.0 (Linux; Android 6.0; HTC M8t Build/MRA58K; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/45.0.2454.95 Mobile Safari/537.36 MMWEBID/4285 MicroMessenger/8.0.15.2001(0x28000F41) Process/tools WeChat/arm32 Weixin GPVersion/1 NetType/WIFI Language/zh_CN ABI/arm32"}: webpSupport,
}
for browser, expected := range testCase {
var header fasthttp.RequestHeader
header.Set("accept", browser[0])
header.Set("user-agent", browser[1])
guessed := guessSupportedFormat(&header)
sort.Strings(expected)
sort.Strings(guessed)
log.Infof("%s expected%s --- actual %s", browser, expected, guessed)
assert.Equal(t, expected, guessed)
}
for browser, is := range testCase {
assert.Equalf(t, is, uaOrigin(browser), "[%v]:[%s]", is, browser)
}
}
func TestHeaderOrigin(t *testing.T) {
// Chrome: Accept: image/avif,image/webp,image/apng,image/*,*/*;q=0.8
// Safari 版本14.0.1 (15610.2.11.51.10, 15610): Accept: image/png,image/svg+xml,image/*;q=0.8,video/*;q=0.8,*/*;q=0.5
// Safari Technology Preview Release 116 (Safari 14.1, WebKit 15611.1.5.3) Accept: image/png,image/svg+xml,image/*;q=0.8,video/*;q=0.8,*/*;q=0.5
// UNLESS YOU KNOW WHAT YOU ARE DOING, DO NOT CHANGE THE TEST CASE MAPPING HERE.
var testCase = map[string]bool{
"image/avif,image/webp,image/apng,image/*,*/*;q=0.8": false,
"*/*": true,
"image/png,image/svg+xml,image/*;q=0.8,video/*;q=0.8,*/*;q=0.5": true,
"I don't know what it is:-)": true,
}
for header, is := range testCase {
assert.Equalf(t, is, headerOrigin(header), "[%v]:[%s]", is, header)
}
}
func TestChanErr(t *testing.T) {
var value = 2
var testC = make(chan int, 2)
testC <- value
chanErr(testC)
value = <-testC
assert.Equal(t, 2, value)
}
func TestGetRemoteImageInfo(t *testing.T) {
@ -197,7 +154,7 @@ func TestFetchRemoteImage(t *testing.T) {
err := fetchRemoteImage(fp, url)
assert.Equal(t, err, nil)
data, _ := ioutil.ReadFile(fp)
assert.Equal(t, "image/x-icon", getFileContentType(data))
assert.Equal(t, "image/vnd.microsoft.icon", getFileContentType(data))
// test can't create file
err = fetchRemoteImage("/", url)

BIN
pics/big.webp Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 65 KiB

BIN
pics/img_over_16383px.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 3.3 MiB

BIN
pics/jpg_without_eoi.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 116 KiB

34
prefetch.go
View File

@ -2,49 +2,49 @@ package main
import (
"fmt"
"github.com/schollz/progressbar/v3"
log "github.com/sirupsen/logrus"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"time"
log "github.com/sirupsen/logrus"
)
func prefetchImages(confImgPath string, ExhaustPath string, QUALITY string) {
var sTime = time.Now()
func prefetchImages(confImgPath string, ExhaustPath string) {
// maximum ongoing prefetch is depending on your core of CPU
var sTime = time.Now()
log.Infof("Prefetching using %d cores", jobs)
var finishChan = make(chan int, jobs)
for i := 0; i < jobs; i++ {
finishChan <- 0
finishChan <- 1
}
//prefetch, recursive through the dir
all := fileCount(confImgPath)
count := 0
var bar = progressbar.Default(all, "Prefetching...")
err := filepath.Walk(confImgPath,
func(picAbsPath string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
return nil
}
// RawImagePath string, ImgFilename string, reqURI string
proposedURI := strings.Replace(picAbsPath, confImgPath, "", 1)
_, p2 := genWebpAbs(picAbsPath, ExhaustPath, info.Name(), proposedURI)
q, _ := strconv.ParseFloat(QUALITY, 32)
_ = os.MkdirAll(path.Dir(p2), 0755)
go webpEncoder(picAbsPath, p2, float32(q), false, finishChan)
count += <-finishChan
//progress bar
_, _ = fmt.Fprintf(os.Stdout, "[Webp Server started] - convert in progress: %d/%d\r", count, all)
avif, webp := genOptimizedAbs(picAbsPath, ExhaustPath, info.Name(), proposedURI)
_ = os.MkdirAll(path.Dir(avif), 0755)
log.Infof("Prefetching %s", picAbsPath)
go convertFilter(picAbsPath, avif, webp, finishChan)
_ = bar.Add(<-finishChan)
return nil
})
if err != nil {
log.Debug(err)
log.Errorln(err)
}
elapsed := time.Since(sTime)
_, _ = fmt.Fprintf(os.Stdout, "Prefetch completeY(^_^)Y\n\n")
_, _ = fmt.Fprintf(os.Stdout, "convert %d file in %s (^_^)Y\n\n", count, elapsed)
_, _ = fmt.Fprintf(os.Stdout, "Prefetch completeY(^_^)Y in %s\n\n", elapsed)
}

15
prefetch_test.go
View File

@ -11,24 +11,15 @@ import (
)
func TestPrefetchImages(t *testing.T) {
// single thread
fp := "./prefetch"
_ = os.Mkdir(fp, 0755)
prefetchImages("./pics", "./prefetch", "80")
prefetchImages("./pics/dir1/", "./prefetch")
count := fileCount("./prefetch")
assert.Equal(t, 8, count)
_ = os.RemoveAll(fp)
// concurrency
jobs = 2
_ = os.Mkdir(fp, 0755)
prefetchImages("./pics", "./prefetch", "80")
count = fileCount("./prefetch")
assert.Equal(t, 6, count)
assert.Equal(t, int64(1), count)
_ = os.RemoveAll(fp)
}
func TestBadPrefetch(t *testing.T) {
jobs = 1
prefetchImages("./pics2", "./prefetch", "80")
prefetchImages("./pics2", "./prefetch")
}

87
router.go
View File

@ -3,15 +3,14 @@ package main
import (
"errors"
"fmt"
"github.com/gofiber/fiber/v2"
log "github.com/sirupsen/logrus"
"io/ioutil"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"github.com/gofiber/fiber/v2"
log "github.com/sirupsen/logrus"
)
func convert(c *fiber.Ctx) error {
@ -24,14 +23,12 @@ func convert(c *fiber.Ctx) error {
rawImageAbs = path.Join(config.ImgPath, reqURI) // /home/xxx/mypic/123.jpg
}
var imgFilename = path.Base(reqURI) // pure filename, 123.jpg
var finalFile string // We'll only need one c.sendFile()
var ua = c.Get("User-Agent")
var accept = c.Get("accept")
log.Debugf("Incoming connection from %s@%s with %s", ua, c.IP(), imgFilename)
log.Debugf("Incoming connection from %s %s", c.IP(), imgFilename)
needOrigin := goOrigin(accept, ua)
if needOrigin {
log.Debugf("A Safari/IE/whatever user has arrived...%s", ua)
goodFormat := guessSupportedFormat(&c.Request().Header)
// old browser only, send the original image or fetch from remote and send.
if len(goodFormat) == 1 {
c.Set("ETag", genEtag(rawImageAbs))
if proxyMode {
localRemoteTmpPath := remoteRaw + reqURI
@ -42,20 +39,7 @@ func convert(c *fiber.Ctx) error {
}
}
// check ext
var allowed = false
for _, ext := range config.AllowedTypes {
haystack := strings.ToLower(imgFilename)
needle := strings.ToLower("." + ext)
if strings.HasSuffix(haystack, needle) {
allowed = true
break
} else {
allowed = false
}
}
if !allowed {
if !checkAllowedType(imgFilename) {
msg := "File extension not allowed! " + imgFilename
log.Warn(msg)
if imageExists(rawImageAbs) {
@ -81,45 +65,26 @@ func convert(c *fiber.Ctx) error {
return errors.New(msg)
}
_, webpAbsPath := genWebpAbs(rawImageAbs, config.ExhaustPath, imgFilename, reqURI)
// generate with timestamp to make sure files are update-to-date
avifAbs, webpAbs := genOptimizedAbs(rawImageAbs, config.ExhaustPath, imgFilename, reqURI)
convertFilter(rawImageAbs, avifAbs, webpAbs, nil)
if imageExists(webpAbsPath) {
finalFile = webpAbsPath
} else {
// we don't have abc.jpg.png1582558990.webp
// delete the old pic and convert a new one.
// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
destHalfFile := path.Clean(path.Join(webpAbsPath, path.Dir(reqURI), imgFilename))
matches, err := filepath.Glob(destHalfFile + "*")
if err != nil {
log.Error(err.Error())
} else {
// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558100.webp <- older ones will be removed
// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp <- keep the latest one
for _, p := range matches {
if strings.Compare(destHalfFile, p) != 0 {
_ = os.Remove(p)
var availableFiles = []string{rawImageAbs}
for _, v := range goodFormat {
if "avif" == v {
availableFiles = append(availableFiles, avifAbs)
}
if "webp" == v {
availableFiles = append(availableFiles, webpAbs)
}
}
//for webp, we need to create dir first
err = os.MkdirAll(path.Dir(webpAbsPath), 0755)
q, _ := strconv.ParseFloat(config.Quality, 32)
err = webpEncoder(rawImageAbs, webpAbsPath, float32(q), true, nil)
if err != nil {
log.Error(err)
_ = c.SendStatus(400)
_ = c.Send([]byte("Bad file. " + err.Error()))
return err
}
finalFile = webpAbsPath
}
var finalFile = findSmallestFiles(availableFiles)
etag := genEtag(finalFile)
c.Set("ETag", etag)
c.Set("X-Compression-Rate", getCompressionRate(rawImageAbs, webpAbsPath))
finalFile = chooseLocalSmallerFile(rawImageAbs, webpAbsPath)
c.Set("X-Compression-Rate", getCompressionRate(rawImageAbs, finalFile))
buf, _ := ioutil.ReadFile(finalFile)
c.Set("content-type", getFileContentType(buf))
return c.SendFile(finalFile)
}
@ -141,12 +106,8 @@ func proxyHandler(c *fiber.Ctx, reqURI string) error {
cleanProxyCache(config.ExhaustPath + reqURI + "*")
localRawImagePath := remoteRaw + reqURI
_ = fetchRemoteImage(localRawImagePath, realRemoteAddr)
q, _ := strconv.ParseFloat(config.Quality, 32)
_ = os.MkdirAll(path.Dir(localEtagWebPPath), 0755)
err := webpEncoder(localRawImagePath, localEtagWebPPath, float32(q), true, nil)
if err != nil {
log.Warning(err)
}
webpEncoder(localRawImagePath, localEtagWebPPath, config.Quality)
chooseProxy(remoteLength, localEtagWebPPath)
return c.SendFile(localEtagWebPPath)
}

45
router_test.go
View File

@ -16,6 +16,8 @@ import (
var (
chromeUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36"
acceptWebP = "image/webp,image/apng,image/*,*/*;q=0.8"
acceptLegacy = "image/jpeg"
safariUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1 Safari/605.1.15"
)
@ -29,10 +31,14 @@ func setupParam() {
remoteRaw = "remote-raw"
}
func requestToServer(url string, app *fiber.App, ua string) (*http.Response, []byte) {
func requestToServer(url string, app *fiber.App, ua, accept string) (*http.Response, []byte) {
req := httptest.NewRequest("GET", url, nil)
req.Header.Set("User-Agent", ua)
resp, _ := app.Test(req, 60000)
req.Header.Set("Accept", accept)
resp, err := app.Test(req, 120000)
if err != nil {
return nil, nil
}
data, _ := ioutil.ReadAll(resp.Body)
return resp, data
}
@ -44,7 +50,7 @@ func TestServerHeaders(t *testing.T) {
url := "http://127.0.0.1:3333/webp_server.bmp"
// test for chrome
response, _ := requestToServer(url, app, chromeUA)
response, _ := requestToServer(url, app, chromeUA, acceptWebP)
ratio := response.Header.Get("X-Compression-Rate")
etag := response.Header.Get("Etag")
@ -52,7 +58,7 @@ func TestServerHeaders(t *testing.T) {
assert.NotEqual(t, "", etag)
// test for safari
response, _ = requestToServer(url, app, safariUA)
response, _ = requestToServer(url, app, safariUA, acceptLegacy)
ratio = response.Header.Get("X-Compression-Rate")
etag = response.Header.Get("Etag")
@ -66,9 +72,9 @@ func TestConvert(t *testing.T) {
"http://127.0.0.1:3333/webp_server.jpg": "image/webp",
"http://127.0.0.1:3333/webp_server.bmp": "image/webp",
"http://127.0.0.1:3333/webp_server.png": "image/webp",
"http://127.0.0.1:3333/empty.jpg": "text/plain; charset=utf-8",
"http://127.0.0.1:3333/empty.jpg": "",
"http://127.0.0.1:3333/png.jpg": "image/webp",
"http://127.0.0.1:3333/12314.jpg": "text/plain; charset=utf-8",
"http://127.0.0.1:3333/12314.jpg": "",
"http://127.0.0.1:3333/dir1/inside.jpg": "image/webp",
"http://127.0.0.1:3333/%e5%a4%aa%e7%a5%9e%e5%95%a6.png": "image/webp",
"http://127.0.0.1:3333/太神啦.png": "image/webp",
@ -78,9 +84,9 @@ func TestConvert(t *testing.T) {
"http://127.0.0.1:3333/webp_server.jpg": "image/jpeg",
"http://127.0.0.1:3333/webp_server.bmp": "image/bmp",
"http://127.0.0.1:3333/webp_server.png": "image/png",
"http://127.0.0.1:3333/empty.jpg": "text/plain; charset=utf-8",
"http://127.0.0.1:3333/empty.jpg": "",
"http://127.0.0.1:3333/png.jpg": "image/png",
"http://127.0.0.1:3333/12314.jpg": "text/plain; charset=utf-8",
"http://127.0.0.1:3333/12314.jpg": "",
"http://127.0.0.1:3333/dir1/inside.jpg": "image/jpeg",
}
@ -89,14 +95,14 @@ func TestConvert(t *testing.T) {
// test Chrome
for url, respType := range testChromeLink {
_, data := requestToServer(url, app, chromeUA)
_, data := requestToServer(url, app, chromeUA, acceptWebP)
contentType := getFileContentType(data)
assert.Equal(t, respType, contentType)
}
// test Safari
for url, respType := range testSafariLink {
_, data := requestToServer(url, app, safariUA)
_, data := requestToServer(url, app, safariUA, acceptLegacy)
contentType := getFileContentType(data)
assert.Equal(t, respType, contentType)
}
@ -112,13 +118,13 @@ func TestConvertNotAllowed(t *testing.T) {
// not allowed, but we have the file
url := "http://127.0.0.1:3333/webp_server.bmp"
_, data := requestToServer(url, app, chromeUA)
_, data := requestToServer(url, app, chromeUA, acceptWebP)
contentType := getFileContentType(data)
assert.Equal(t, "image/bmp", contentType)
// not allowed, random file
url = url + "hagdgd"
_, data = requestToServer(url, app, chromeUA)
_, data = requestToServer(url, app, chromeUA, acceptWebP)
assert.Contains(t, string(data), "File extension not allowed")
}
@ -132,7 +138,7 @@ func TestConvertProxyModeBad(t *testing.T) {
// this is local random image, should be 500
url := "http://127.0.0.1:3333/webp_8888server.bmp"
resp, _ := requestToServer(url, app, chromeUA)
resp, _ := requestToServer(url, app, chromeUA, acceptWebP)
assert.Equal(t, http.StatusInternalServerError, resp.StatusCode)
}
@ -147,29 +153,26 @@ func TestConvertProxyModeWork(t *testing.T) {
config.ImgPath = "https://webp.sh"
url := "https://webp.sh/images/cover.jpg"
resp, data := requestToServer(url, app, chromeUA)
resp, data := requestToServer(url, app, chromeUA, acceptWebP)
assert.Equal(t, http.StatusOK, resp.StatusCode)
assert.Equal(t, "image/webp", getFileContentType(data))
// test proxyMode with Safari
resp, data = requestToServer(url, app, safariUA)
resp, data = requestToServer(url, app, safariUA, acceptLegacy)
assert.Equal(t, http.StatusOK, resp.StatusCode)
assert.Equal(t, "image/jpeg", getFileContentType(data))
}
func TestConvertBigger(t *testing.T) {
setupParam()
config.Quality = "100"
jpg, _ := ioutil.ReadFile("pics/big.jpg")
config.Quality = 100
var app = fiber.New()
app.Get("/*", convert)
url := "http://127.0.0.1:3333/big.jpg"
response, data := requestToServer(url, app, chromeUA)
response, data := requestToServer(url, app, chromeUA, acceptWebP)
assert.Equal(t, "image/jpeg", response.Header.Get("content-type"))
assert.True(t, len(data) == len(jpg))
assert.Equal(t, "image/jpeg", getFileContentType(data))
_ = os.RemoveAll(config.ExhaustPath)
}

17
update.go
View File

@ -3,13 +3,11 @@ package main
import (
"encoding/json"
"fmt"
log "github.com/sirupsen/logrus"
"github.com/staktrace/go-update"
"io/ioutil"
"net/http"
"os"
"path"
"runtime"
log "github.com/sirupsen/logrus"
)
func autoUpdate() {
@ -47,12 +45,13 @@ func autoUpdate() {
log.Debugf("%s-%s not found on release.", runtime.GOOS, runtime.GOARCH)
return
}
data, _ := ioutil.ReadAll(resp.Body)
_ = os.Mkdir("update", 0755)
err := ioutil.WriteFile(path.Join("update", filename), data, 0755)
if err == nil {
log.Info("Update complete. Please find your binary from update directory.")
err := update.Apply(resp.Body, update.Options{})
if err != nil {
// error handling
log.Errorf("Update error. %v", err)
} else {
log.Info("Update complete. Please restart to apply changes.")
}
_ = resp.Body.Close()
}

2
update_test.go
View File

@ -23,7 +23,7 @@ func Test404AutoUpdate(t *testing.T) {
dir := "./update"
releaseUrl = releaseUrl + "a"
autoUpdate()
assert.Equal(t, 0, fileCount(dir))
assert.Equal(t, int64(0), fileCount(dir))
_ = os.RemoveAll(dir)
}

4
webp-server.go
View File

@ -41,7 +41,7 @@ func deferInit() {
FullTimestamp: true,
TimestampFormat: "2006-01-02 15:04:05",
CallerPrettyfier: func(f *runtime.Frame) (string, string) {
return fmt.Sprintf("[%s()]", f.Function), ""
return fmt.Sprintf("[%d:%s()]", f.Line, f.Function), ""
},
}
log.SetFormatter(Formatter)
@ -99,7 +99,7 @@ Develop by WebP Server team. https://github.com/webp-sh`, version)
switchProxyMode()
if prefetch {
go prefetchImages(config.ImgPath, config.ExhaustPath, config.Quality)
go prefetchImages(config.ImgPath, config.ExhaustPath)
}
app := fiber.New(fiber.Config{

4
webp-server_test.go
View File

@ -21,7 +21,7 @@ func TestLoadConfig(t *testing.T) {
assert.Equal(t, "./exhaust", c.ExhaustPath)
assert.Equal(t, "127.0.0.1", c.Host)
assert.Equal(t, "3333", c.Port)
assert.Equal(t, "80", c.Quality)
assert.Equal(t, float32(80), c.Quality)
assert.Equal(t, "./pics", c.ImgPath)
assert.Equal(t, []string{"jpg", "png", "jpeg", "bmp"}, c.AllowedTypes)
}
@ -43,7 +43,7 @@ func TestMainFunction(t *testing.T) {
// run main function
go main()
time.Sleep(time.Second * 2)
time.Sleep(time.Second * 5)
// verbose, prefetch
assert.Equal(t, log.GetLevel(), log.DebugLevel)
assert.True(t, verboseMode)