wood/webp_server_go
1
0
Fork 0
mirror of https://github.com/woodchen-ink/webp_server_go.git synced 2025-07-18 13:42:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

Support extra parameters on request (#192)

Browse Source 
* version 1

* some update

* Fix ST1020

* Remove ST1020

* Fix CI

* Fix params

* Fix order

* Allow reqURIwithQuery pass through when proxy mode

* Fix bugs

* Extract resizeImage

* Check for err
This commit is contained in:
Nova Kwok 2023-05-15 20:13:00 +08:00 committed by GitHub
parent bfa8aae10c
commit bcb3f1452b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 150 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
config.go
View File

@ -1,5 +1,7 @@
package main
import "fmt"
type Config struct {
Host string `json:"HOST"`
Port string `json:"PORT"`
@ -8,6 +10,17 @@ type Config struct {
AllowedTypes []string `json:"ALLOWED_TYPES"`
ExhaustPath string `json:"EXHAUST_PATH"`
EnableAVIF bool `json:"ENABLE_AVIF"`
EnableExtraParams bool `json:"ENABLE_EXTRA_PARAMS"`
}
type ExtraParams struct {
Width int // in px
Height int // in px
}
// String : convert ExtraParams to string, used to generate cache path
func (e *ExtraParams) String() string {
return fmt.Sprintf("_width=%d&height=%d", e.Width, e.Height)
}
var (
@ -18,7 +31,7 @@ var (
prefetch, proxyMode bool
remoteRaw = "remote-raw"
config Config
version = "0.6.0"
version = "0.7.0"
)
const (
@ -30,7 +43,8 @@ const (
"IMG_PATH": "./pics",
"EXHAUST_PATH": "./exhaust",
"ALLOWED_TYPES": ["jpg","png","jpeg","bmp"],
"ENABLE_AVIF": false
"ENABLE_AVIF": false,
"ENABLE_EXTRA_PARAMS": false
}`
sampleSystemd = `

3
config.json
View File

@ -5,5 +5,6 @@
"IMG_PATH": "./pics",
"EXHAUST_PATH": "./exhaust",
"ALLOWED_TYPES": ["jpg","png","jpeg","bmp"],
"ENABLE_AVIF": false
"ENABLE_AVIF": false,
"ENABLE_EXTRA_PARAMS": false
}

61
encoder.go
View File

@ -12,14 +12,35 @@ import (
log "github.com/sirupsen/logrus"
)
func convertFilter(raw, avifPath, webpPath string, c chan int) {
func resizeImage(img *vips.ImageRef, extraParams ExtraParams) error {
imgHeightWidthRatio := float32(img.Metadata().Height) / float32(img.Metadata().Width)
if extraParams.Width > 0 && extraParams.Height > 0 {
err := img.Thumbnail(extraParams.Width, extraParams.Height, 0)
if err != nil {
return err
}
} else if extraParams.Width > 0 && extraParams.Height == 0 {
err := img.Thumbnail(extraParams.Width, int(float32(extraParams.Width)*imgHeightWidthRatio), 0)
if err != nil {
return err
}
} else if extraParams.Height > 0 && extraParams.Width == 0 {
err := img.Thumbnail(int(float32(extraParams.Height)/imgHeightWidthRatio), extraParams.Height, 0)
if err != nil {
return err
}
}
return nil
}
func convertFilter(raw, avifPath, webpPath string, extraParams ExtraParams, c chan int) {
// all absolute paths
var wg sync.WaitGroup
wg.Add(2)
if !imageExists(avifPath) && config.EnableAVIF {
go func() {
err := convertImage(raw, avifPath, "avif")
err := convertImage(raw, avifPath, "avif", extraParams)
if err != nil {
log.Errorln(err)
}
@ -31,7 +52,7 @@ func convertFilter(raw, avifPath, webpPath string, c chan int) {
if !imageExists(webpPath) {
go func() {
err := convertImage(raw, webpPath, "webp")
err := convertImage(raw, webpPath, "webp", extraParams)
if err != nil {
log.Errorln(err)
}
@ -47,11 +68,12 @@ func convertFilter(raw, avifPath, webpPath string, c chan int) {
}
}
func convertImage(raw, optimized, itype string) error {
// we don't have abc.jpg.png1582558990.webp
// delete the old pic and convert a new one.
func convertImage(raw, optimized, itype string, extraParams ExtraParams) error {
// we don't have /path/to/tsuki.jpg.1582558990.webp, maybe we have /path/to/tsuki.jpg.1082008000.webp
// delete the old converted pic and convert a new one.
// optimized: /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
// we'll delete file starts with /home/webp_server/exhaust/path/to/tsuki.jpg.ts.itype
// If contain extraParams like tsuki.jpg?width=200, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp?width=200
s := strings.Split(path.Base(optimized), ".")
pattern := path.Join(path.Dir(optimized), s[0]+"."+s[1]+".*."+s[len(s)-1])
@ -73,24 +95,33 @@ func convertImage(raw, optimized, itype string) error {
switch itype {
case "webp":
err = webpEncoder(raw, optimized, config.Quality)
err = webpEncoder(raw, optimized, config.Quality, extraParams)
case "avif":
err = avifEncoder(raw, optimized, config.Quality)
err = avifEncoder(raw, optimized, config.Quality, extraParams)
}
return err
}
func avifEncoder(p1, p2 string, quality int) error {
func avifEncoder(p1, p2 string, quality int, extraParams ExtraParams) error {
// if convert fails, return error; success nil
var buf []byte
img, err := vips.NewImageFromFile(p1)
if err != nil {
return err
}
if config.EnableExtraParams {
err = resizeImage(img, extraParams)
if err != nil {
return err
}
}
// AVIF has a maximum resolution of 65536 x 65536 pixels.
if img.Metadata().Width > avifMax || img.Metadata().Height > avifMax {
return errors.New("WebP: image too large")
return errors.New("AVIF: image too large")
}
// If quality >= 100, we use lossless mode
if quality >= 100 {
buf, _, err = img.ExportAvif(&vips.AvifExportParams{
@ -118,7 +149,7 @@ func avifEncoder(p1, p2 string, quality int) error {
return nil
}
func webpEncoder(p1, p2 string, quality int) error {
func webpEncoder(p1, p2 string, quality int, extraParams ExtraParams) error {
// if convert fails, return error; success nil
var buf []byte
img, err := vips.NewImageFromFile(p1)
@ -126,10 +157,18 @@ func webpEncoder(p1, p2 string, quality int) error {
return err
}
if config.EnableExtraParams {
err = resizeImage(img, extraParams)
if err != nil {
return err
}
}
// The maximum pixel dimensions of a WebP image is 16383 x 16383.
if img.Metadata().Width > webpMax || img.Metadata().Height > webpMax {
return errors.New("WebP: image too large")
}
// If quality >= 100, we use lossless mode
if quality >= 100 {
buf, _, err = img.ExportWebp(&vips.WebpExportParams{

12
encoder_test.go
View File

@ -34,25 +34,25 @@ func TestWebPEncoder(t *testing.T) {
func TestAvifEncoder(t *testing.T) {
// Only one file: img_over_16383px.jpg might cause memory issues on CI environment
assert.Nil(t, avifEncoder("./pics/big.jpg", dest, 80))
assert.Nil(t, avifEncoder("./pics/big.jpg", dest, 80, ExtraParams{Width: 0, Height: 0}))
assertType(t, dest, "image/avif")
}
func TestNonExistImage(t *testing.T) {
assert.NotNil(t, webpEncoder("./pics/empty.jpg", dest, 80))
assert.NotNil(t, avifEncoder("./pics/empty.jpg", dest, 80))
assert.NotNil(t, webpEncoder("./pics/empty.jpg", dest, 80, ExtraParams{Width: 0, Height: 0}))
assert.NotNil(t, avifEncoder("./pics/empty.jpg", dest, 80, ExtraParams{Width: 0, Height: 0}))
}
func TestHighResolutionImage(t *testing.T) {
assert.NotNil(t, webpEncoder("./pics/img_over_16383px.jpg", dest, 80))
assert.Nil(t, avifEncoder("./pics/img_over_16383px.jpg", dest, 80))
assert.NotNil(t, webpEncoder("./pics/img_over_16383px.jpg", dest, 80, ExtraParams{Width: 0, Height: 0}))
assert.Nil(t, avifEncoder("./pics/img_over_16383px.jpg", dest, 80, ExtraParams{Width: 0, Height: 0}))
}
func runEncoder(t *testing.T, file string, dest string) {
if file == "pics/empty.jpg" {
t.Log("Empty file, that's okay.")
}
_ = webpEncoder(file, dest, 80)
_ = webpEncoder(file, dest, 80, ExtraParams{Width: 0, Height: 0})
assertType(t, dest, "image/webp")
}

20
helper.go
View File

@ -2,6 +2,8 @@ package main
import (
"bytes"
"crypto/sha1" //#nosec
"encoding/hex"
"fmt"
"hash/crc32"
"io"
@ -136,7 +138,7 @@ func cleanProxyCache(cacheImagePath string) {
}
}
func genOptimizedAbsPath(rawImagePath string, exhaustPath string, imageName string, reqURI string) (string, string) {
func genOptimizedAbsPath(rawImagePath string, exhaustPath string, imageName string, reqURI string, extraParams ExtraParams) (string, string) {
// get file mod time
STAT, err := os.Stat(rawImagePath)
if err != nil {
@ -149,6 +151,14 @@ func genOptimizedAbsPath(rawImagePath string, exhaustPath string, imageName stri
// avifFilename: abc.jpg.png -> abc.jpg.png.1582558990.avif
avifFilename := fmt.Sprintf("%s.%d.avif", imageName, ModifiedTime)
// If extraParams not enabled, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
// If extraParams enabled, and given request at tsuki.jpg?width=200, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=200&height=0
// If extraParams enabled, and given request at tsuki.jpg, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=0&height=0
if config.EnableExtraParams {
webpFilename = webpFilename + extraParams.String()
avifFilename = avifFilename + extraParams.String()
}
// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
// Custom Exhaust: /path/to/exhaust/web_path/web_to/tsuki.jpg.1582558990.webp
webpAbsolutePath := path.Clean(path.Join(exhaustPath, path.Dir(reqURI), webpFilename))
@ -216,7 +226,6 @@ func guessSupportedFormat(header *fasthttp.RequestHeader) []string {
}
}
return accepted
}
func chooseProxy(proxyRawSize string, optimizedAbs string) bool {
@ -246,3 +255,10 @@ func findSmallestFiles(files []string) string {
}
return final
}
func Sha1Path(uri string) string {
/* #nosec */
h := sha1.New()
h.Write([]byte(uri))
return hex.EncodeToString(h.Sum(nil))
}

2
helper_test.go
View File

@ -47,7 +47,7 @@ func TestImageExists(t *testing.T) {
func TestGenWebpAbs(t *testing.T) {
cwd, cooked := genOptimizedAbsPath("./pics/webp_server.png", "/tmp",
"test", "a")
"test", "a", ExtraParams{Width: 0, Height: 0})
if !strings.Contains(cwd, "webp_server_go") {
t.Logf("Result: [%v], Expected: [%v]", cwd, "webp_server_go")
}

4
prefetch.go
View File

@ -34,10 +34,10 @@ func prefetchImages(confImgPath string, ExhaustPath string) {
}
// RawImagePath string, ImgFilename string, reqURI string
proposedURI := strings.Replace(picAbsPath, confImgPath, "", 1)
avif, webp := genOptimizedAbsPath(picAbsPath, ExhaustPath, info.Name(), proposedURI)
avif, webp := genOptimizedAbsPath(picAbsPath, ExhaustPath, info.Name(), proposedURI, ExtraParams{Width: 0, Height: 0})
_ = os.MkdirAll(path.Dir(avif), 0755)
log.Infof("Prefetching %s", picAbsPath)
go convertFilter(picAbsPath, avif, webp, finishChan)
go convertFilter(picAbsPath, avif, webp, ExtraParams{Width: 0, Height: 0}, finishChan)
_ = bar.Add(<-finishChan)
return nil
})

61
router.go
View File

@ -7,6 +7,7 @@ import (
"net/url"
"os"
"path"
"strconv"
"github.com/gofiber/fiber/v2"
log "github.com/sirupsen/logrus"
@ -15,13 +16,38 @@ import (
func convert(c *fiber.Ctx) error {
//basic vars
var reqURI, _ = url.QueryUnescape(c.Path()) // /mypic/123.jpg
var reqURIwithQuery, _ = url.QueryUnescape(c.OriginalURL()) // /mypic/123.jpg?someother=200&somebugs=200
// Sometimes reqURIwithQuery can be https://example.tld/mypic/123.jpg?someother=200&somebugs=200, we need to extract it.
u, err := url.Parse(reqURIwithQuery)
if err != nil {
log.Errorln(err)
}
reqURIwithQuery = u.RequestURI()
// delete ../ in reqURI to mitigate directory traversal
reqURI = path.Clean(reqURI)
reqURIwithQuery = path.Clean(reqURIwithQuery)
// Begin Extra params
var extraParams ExtraParams
Width := c.Query("width")
Height := c.Query("height")
WidthInt, err := strconv.Atoi(Width)
if err != nil {
WidthInt = 0
}
HeightInt, err := strconv.Atoi(Height)
if err != nil {
HeightInt = 0
}
extraParams = ExtraParams{
Width: WidthInt,
Height: HeightInt,
}
// End Extra params
var rawImageAbs string
if proxyMode {
rawImageAbs = config.ImgPath + reqURI
rawImageAbs = config.ImgPath + reqURIwithQuery // https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200
} else {
rawImageAbs = path.Join(config.ImgPath, reqURI) // /home/xxx/mypic/123.jpg
}
@ -51,7 +77,7 @@ func convert(c *fiber.Ctx) error {
}
if proxyMode {
return proxyHandler(c, reqURI)
return proxyHandler(c, reqURIwithQuery)
}
// Check the original image for existence,
@ -64,8 +90,11 @@ func convert(c *fiber.Ctx) error {
}
// generate with timestamp to make sure files are update-to-date
avifAbs, webpAbs := genOptimizedAbsPath(rawImageAbs, config.ExhaustPath, imgFilename, reqURI)
convertFilter(rawImageAbs, avifAbs, webpAbs, nil)
// If extraParams not enabled, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
// If extraParams enabled, and given request at tsuki.jpg?width=200, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=200&height=0
// If extraParams enabled, and given request at tsuki.jpg, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=0&height=0
avifAbs, webpAbs := genOptimizedAbsPath(rawImageAbs, config.ExhaustPath, imgFilename, reqURI, extraParams)
convertFilter(rawImageAbs, avifAbs, webpAbs, extraParams, nil)
var availableFiles = []string{rawImageAbs}
for _, v := range goodFormat {
@ -91,25 +120,29 @@ func convert(c *fiber.Ctx) error {
return c.SendFile(finalFileName)
}
func proxyHandler(c *fiber.Ctx, reqURI string) error {
// https://test.webp.sh/node.png
realRemoteAddr := config.ImgPath + reqURI
func proxyHandler(c *fiber.Ctx, reqURIwithQuery string) error {
// https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200
realRemoteAddr := config.ImgPath + reqURIwithQuery
// Since we cannot store file in format of "mypic/123.jpg?someother=200&somebugs=200", we need to hash it.
reqURIwithQueryHash := Sha1Path(reqURIwithQuery) // 378e740ca56144b7587f3af9debeee544842879a
localRawImagePath := remoteRaw + "/" + reqURIwithQueryHash // For store the remote raw image, /home/webp_server/remote-raw/378e740ca56144b7587f3af9debeee544842879a
// Ping Remote for status code and etag info
log.Infof("Remote Addr is %s fetching", realRemoteAddr)
statusCode, etagValue, remoteLength := getRemoteImageInfo(realRemoteAddr)
localEtagWebPPath := config.ExhaustPath + "/" + reqURIwithQueryHash + "-etag-" + etagValue // For store the remote webp image, /home/webp_server/exhaust/378e740ca56144b7587f3af9debeee544842879a-etag-<etagValue>
if statusCode == 200 {
// Check local path: /node.png-etag-<etagValue>
localEtagWebPPath := config.ExhaustPath + reqURI + "-etag-" + etagValue
if imageExists(localEtagWebPPath) {
chooseProxy(remoteLength, localEtagWebPPath)
return c.SendFile(localEtagWebPPath)
} else {
// Temporary store of remote file.
cleanProxyCache(config.ExhaustPath + reqURI + "*")
localRawImagePath := remoteRaw + reqURI
cleanProxyCache(config.ExhaustPath + reqURIwithQuery + "*")
_ = fetchRemoteImage(localRawImagePath, realRemoteAddr)
_ = os.MkdirAll(path.Dir(localEtagWebPPath), 0755)
encodeErr := webpEncoder(localRawImagePath, localEtagWebPPath, config.Quality)
encodeErr := webpEncoder(localRawImagePath, localEtagWebPPath, config.Quality, ExtraParams{Width: 0, Height: 0})
if encodeErr != nil {
// Send as it is.
return c.SendFile(localRawImagePath)
@ -122,7 +155,7 @@ func proxyHandler(c *fiber.Ctx, reqURI string) error {
_ = c.Send([]byte(msg))
log.Warn(msg)
_ = c.SendStatus(statusCode)
cleanProxyCache(config.ExhaustPath + reqURI + "*")
cleanProxyCache(config.ExhaustPath + reqURIwithQuery + "*")
return errors.New(msg)
}
}