From bcb3f1452ba894f6c7d37afa7df0704b76813c24 Mon Sep 17 00:00:00 2001 From: Nova Kwok Date: Mon, 15 May 2023 20:13:00 +0800 Subject: [PATCH] Support extra parameters on request (#192) * version 1 * some update * Fix ST1020 * Remove ST1020 * Fix CI * Fix params * Fix order * Allow reqURIwithQuery pass through when proxy mode * Fix bugs * Extract resizeImage * Check for err --- config.go | 32 ++++++++++++++++++------- config.json | 3 ++- encoder.go | 61 ++++++++++++++++++++++++++++++++++++++--------- encoder_test.go | 12 +++++----- helper.go | 20 ++++++++++++++-- helper_test.go | 2 +- prefetch.go | 4 ++-- router.go | 63 +++++++++++++++++++++++++++++++++++++------------ 8 files changed, 150 insertions(+), 47 deletions(-) diff --git a/config.go b/config.go index ae63a3d..246bdb1 100644 --- a/config.go +++ b/config.go @@ -1,13 +1,26 @@ package main +import "fmt" + type Config struct { - Host string `json:"HOST"` - Port string `json:"PORT"` - ImgPath string `json:"IMG_PATH"` - Quality int `json:"QUALITY,string"` - AllowedTypes []string `json:"ALLOWED_TYPES"` - ExhaustPath string `json:"EXHAUST_PATH"` - EnableAVIF bool `json:"ENABLE_AVIF"` + Host string `json:"HOST"` + Port string `json:"PORT"` + ImgPath string `json:"IMG_PATH"` + Quality int `json:"QUALITY,string"` + AllowedTypes []string `json:"ALLOWED_TYPES"` + ExhaustPath string `json:"EXHAUST_PATH"` + EnableAVIF bool `json:"ENABLE_AVIF"` + EnableExtraParams bool `json:"ENABLE_EXTRA_PARAMS"` +} + +type ExtraParams struct { + Width int // in px + Height int // in px +} + +// String : convert ExtraParams to string, used to generate cache path +func (e *ExtraParams) String() string { + return fmt.Sprintf("_width=%d&height=%d", e.Width, e.Height) } var ( @@ -18,7 +31,7 @@ var ( prefetch, proxyMode bool remoteRaw = "remote-raw" config Config - version = "0.6.0" + version = "0.7.0" ) const ( @@ -30,7 +43,8 @@ const ( "IMG_PATH": "./pics", "EXHAUST_PATH": "./exhaust", "ALLOWED_TYPES": ["jpg","png","jpeg","bmp"], - "ENABLE_AVIF": false + "ENABLE_AVIF": false, + "ENABLE_EXTRA_PARAMS": false }` sampleSystemd = ` diff --git a/config.json b/config.json index 24805a9..7f09d54 100644 --- a/config.json +++ b/config.json @@ -5,5 +5,6 @@ "IMG_PATH": "./pics", "EXHAUST_PATH": "./exhaust", "ALLOWED_TYPES": ["jpg","png","jpeg","bmp"], - "ENABLE_AVIF": false + "ENABLE_AVIF": false, + "ENABLE_EXTRA_PARAMS": false } \ No newline at end of file diff --git a/encoder.go b/encoder.go index ab3a130..69f2be0 100644 --- a/encoder.go +++ b/encoder.go @@ -12,14 +12,35 @@ import ( log "github.com/sirupsen/logrus" ) -func convertFilter(raw, avifPath, webpPath string, c chan int) { +func resizeImage(img *vips.ImageRef, extraParams ExtraParams) error { + imgHeightWidthRatio := float32(img.Metadata().Height) / float32(img.Metadata().Width) + if extraParams.Width > 0 && extraParams.Height > 0 { + err := img.Thumbnail(extraParams.Width, extraParams.Height, 0) + if err != nil { + return err + } + } else if extraParams.Width > 0 && extraParams.Height == 0 { + err := img.Thumbnail(extraParams.Width, int(float32(extraParams.Width)*imgHeightWidthRatio), 0) + if err != nil { + return err + } + } else if extraParams.Height > 0 && extraParams.Width == 0 { + err := img.Thumbnail(int(float32(extraParams.Height)/imgHeightWidthRatio), extraParams.Height, 0) + if err != nil { + return err + } + } + return nil +} + +func convertFilter(raw, avifPath, webpPath string, extraParams ExtraParams, c chan int) { // all absolute paths var wg sync.WaitGroup wg.Add(2) if !imageExists(avifPath) && config.EnableAVIF { go func() { - err := convertImage(raw, avifPath, "avif") + err := convertImage(raw, avifPath, "avif", extraParams) if err != nil { log.Errorln(err) } @@ -31,7 +52,7 @@ func convertFilter(raw, avifPath, webpPath string, c chan int) { if !imageExists(webpPath) { go func() { - err := convertImage(raw, webpPath, "webp") + err := convertImage(raw, webpPath, "webp", extraParams) if err != nil { log.Errorln(err) } @@ -47,11 +68,12 @@ func convertFilter(raw, avifPath, webpPath string, c chan int) { } } -func convertImage(raw, optimized, itype string) error { - // we don't have abc.jpg.png1582558990.webp - // delete the old pic and convert a new one. +func convertImage(raw, optimized, itype string, extraParams ExtraParams) error { + // we don't have /path/to/tsuki.jpg.1582558990.webp, maybe we have /path/to/tsuki.jpg.1082008000.webp + // delete the old converted pic and convert a new one. // optimized: /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp // we'll delete file starts with /home/webp_server/exhaust/path/to/tsuki.jpg.ts.itype + // If contain extraParams like tsuki.jpg?width=200, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp?width=200 s := strings.Split(path.Base(optimized), ".") pattern := path.Join(path.Dir(optimized), s[0]+"."+s[1]+".*."+s[len(s)-1]) @@ -73,24 +95,33 @@ func convertImage(raw, optimized, itype string) error { switch itype { case "webp": - err = webpEncoder(raw, optimized, config.Quality) + err = webpEncoder(raw, optimized, config.Quality, extraParams) case "avif": - err = avifEncoder(raw, optimized, config.Quality) + err = avifEncoder(raw, optimized, config.Quality, extraParams) } return err } -func avifEncoder(p1, p2 string, quality int) error { +func avifEncoder(p1, p2 string, quality int, extraParams ExtraParams) error { // if convert fails, return error; success nil var buf []byte img, err := vips.NewImageFromFile(p1) if err != nil { return err } + + if config.EnableExtraParams { + err = resizeImage(img, extraParams) + if err != nil { + return err + } + } + // AVIF has a maximum resolution of 65536 x 65536 pixels. if img.Metadata().Width > avifMax || img.Metadata().Height > avifMax { - return errors.New("WebP: image too large") + return errors.New("AVIF: image too large") } + // If quality >= 100, we use lossless mode if quality >= 100 { buf, _, err = img.ExportAvif(&vips.AvifExportParams{ @@ -118,7 +149,7 @@ func avifEncoder(p1, p2 string, quality int) error { return nil } -func webpEncoder(p1, p2 string, quality int) error { +func webpEncoder(p1, p2 string, quality int, extraParams ExtraParams) error { // if convert fails, return error; success nil var buf []byte img, err := vips.NewImageFromFile(p1) @@ -126,10 +157,18 @@ func webpEncoder(p1, p2 string, quality int) error { return err } + if config.EnableExtraParams { + err = resizeImage(img, extraParams) + if err != nil { + return err + } + } + // The maximum pixel dimensions of a WebP image is 16383 x 16383. if img.Metadata().Width > webpMax || img.Metadata().Height > webpMax { return errors.New("WebP: image too large") } + // If quality >= 100, we use lossless mode if quality >= 100 { buf, _, err = img.ExportWebp(&vips.WebpExportParams{ diff --git a/encoder_test.go b/encoder_test.go index cca8744..b27bf42 100644 --- a/encoder_test.go +++ b/encoder_test.go @@ -34,25 +34,25 @@ func TestWebPEncoder(t *testing.T) { func TestAvifEncoder(t *testing.T) { // Only one file: img_over_16383px.jpg might cause memory issues on CI environment - assert.Nil(t, avifEncoder("./pics/big.jpg", dest, 80)) + assert.Nil(t, avifEncoder("./pics/big.jpg", dest, 80, ExtraParams{Width: 0, Height: 0})) assertType(t, dest, "image/avif") } func TestNonExistImage(t *testing.T) { - assert.NotNil(t, webpEncoder("./pics/empty.jpg", dest, 80)) - assert.NotNil(t, avifEncoder("./pics/empty.jpg", dest, 80)) + assert.NotNil(t, webpEncoder("./pics/empty.jpg", dest, 80, ExtraParams{Width: 0, Height: 0})) + assert.NotNil(t, avifEncoder("./pics/empty.jpg", dest, 80, ExtraParams{Width: 0, Height: 0})) } func TestHighResolutionImage(t *testing.T) { - assert.NotNil(t, webpEncoder("./pics/img_over_16383px.jpg", dest, 80)) - assert.Nil(t, avifEncoder("./pics/img_over_16383px.jpg", dest, 80)) + assert.NotNil(t, webpEncoder("./pics/img_over_16383px.jpg", dest, 80, ExtraParams{Width: 0, Height: 0})) + assert.Nil(t, avifEncoder("./pics/img_over_16383px.jpg", dest, 80, ExtraParams{Width: 0, Height: 0})) } func runEncoder(t *testing.T, file string, dest string) { if file == "pics/empty.jpg" { t.Log("Empty file, that's okay.") } - _ = webpEncoder(file, dest, 80) + _ = webpEncoder(file, dest, 80, ExtraParams{Width: 0, Height: 0}) assertType(t, dest, "image/webp") } diff --git a/helper.go b/helper.go index 8cb6b4e..29629c5 100644 --- a/helper.go +++ b/helper.go @@ -2,6 +2,8 @@ package main import ( "bytes" + "crypto/sha1" //#nosec + "encoding/hex" "fmt" "hash/crc32" "io" @@ -136,7 +138,7 @@ func cleanProxyCache(cacheImagePath string) { } } -func genOptimizedAbsPath(rawImagePath string, exhaustPath string, imageName string, reqURI string) (string, string) { +func genOptimizedAbsPath(rawImagePath string, exhaustPath string, imageName string, reqURI string, extraParams ExtraParams) (string, string) { // get file mod time STAT, err := os.Stat(rawImagePath) if err != nil { @@ -149,6 +151,14 @@ func genOptimizedAbsPath(rawImagePath string, exhaustPath string, imageName stri // avifFilename: abc.jpg.png -> abc.jpg.png.1582558990.avif avifFilename := fmt.Sprintf("%s.%d.avif", imageName, ModifiedTime) + // If extraParams not enabled, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp + // If extraParams enabled, and given request at tsuki.jpg?width=200, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=200&height=0 + // If extraParams enabled, and given request at tsuki.jpg, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=0&height=0 + if config.EnableExtraParams { + webpFilename = webpFilename + extraParams.String() + avifFilename = avifFilename + extraParams.String() + } + // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp // Custom Exhaust: /path/to/exhaust/web_path/web_to/tsuki.jpg.1582558990.webp webpAbsolutePath := path.Clean(path.Join(exhaustPath, path.Dir(reqURI), webpFilename)) @@ -216,7 +226,6 @@ func guessSupportedFormat(header *fasthttp.RequestHeader) []string { } } return accepted - } func chooseProxy(proxyRawSize string, optimizedAbs string) bool { @@ -246,3 +255,10 @@ func findSmallestFiles(files []string) string { } return final } + +func Sha1Path(uri string) string { + /* #nosec */ + h := sha1.New() + h.Write([]byte(uri)) + return hex.EncodeToString(h.Sum(nil)) +} diff --git a/helper_test.go b/helper_test.go index 8aced5b..eb50e6c 100644 --- a/helper_test.go +++ b/helper_test.go @@ -47,7 +47,7 @@ func TestImageExists(t *testing.T) { func TestGenWebpAbs(t *testing.T) { cwd, cooked := genOptimizedAbsPath("./pics/webp_server.png", "/tmp", - "test", "a") + "test", "a", ExtraParams{Width: 0, Height: 0}) if !strings.Contains(cwd, "webp_server_go") { t.Logf("Result: [%v], Expected: [%v]", cwd, "webp_server_go") } diff --git a/prefetch.go b/prefetch.go index 654b29e..67c2584 100644 --- a/prefetch.go +++ b/prefetch.go @@ -34,10 +34,10 @@ func prefetchImages(confImgPath string, ExhaustPath string) { } // RawImagePath string, ImgFilename string, reqURI string proposedURI := strings.Replace(picAbsPath, confImgPath, "", 1) - avif, webp := genOptimizedAbsPath(picAbsPath, ExhaustPath, info.Name(), proposedURI) + avif, webp := genOptimizedAbsPath(picAbsPath, ExhaustPath, info.Name(), proposedURI, ExtraParams{Width: 0, Height: 0}) _ = os.MkdirAll(path.Dir(avif), 0755) log.Infof("Prefetching %s", picAbsPath) - go convertFilter(picAbsPath, avif, webp, finishChan) + go convertFilter(picAbsPath, avif, webp, ExtraParams{Width: 0, Height: 0}, finishChan) _ = bar.Add(<-finishChan) return nil }) diff --git a/router.go b/router.go index 20cea72..534b704 100644 --- a/router.go +++ b/router.go @@ -7,6 +7,7 @@ import ( "net/url" "os" "path" + "strconv" "github.com/gofiber/fiber/v2" log "github.com/sirupsen/logrus" @@ -14,14 +15,39 @@ import ( func convert(c *fiber.Ctx) error { //basic vars - var reqURI, _ = url.QueryUnescape(c.Path()) // /mypic/123.jpg - + var reqURI, _ = url.QueryUnescape(c.Path()) // /mypic/123.jpg + var reqURIwithQuery, _ = url.QueryUnescape(c.OriginalURL()) // /mypic/123.jpg?someother=200&somebugs=200 + // Sometimes reqURIwithQuery can be https://example.tld/mypic/123.jpg?someother=200&somebugs=200, we need to extract it. + u, err := url.Parse(reqURIwithQuery) + if err != nil { + log.Errorln(err) + } + reqURIwithQuery = u.RequestURI() // delete ../ in reqURI to mitigate directory traversal reqURI = path.Clean(reqURI) + reqURIwithQuery = path.Clean(reqURIwithQuery) + + // Begin Extra params + var extraParams ExtraParams + Width := c.Query("width") + Height := c.Query("height") + WidthInt, err := strconv.Atoi(Width) + if err != nil { + WidthInt = 0 + } + HeightInt, err := strconv.Atoi(Height) + if err != nil { + HeightInt = 0 + } + extraParams = ExtraParams{ + Width: WidthInt, + Height: HeightInt, + } + // End Extra params var rawImageAbs string if proxyMode { - rawImageAbs = config.ImgPath + reqURI + rawImageAbs = config.ImgPath + reqURIwithQuery // https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200 } else { rawImageAbs = path.Join(config.ImgPath, reqURI) // /home/xxx/mypic/123.jpg } @@ -51,7 +77,7 @@ func convert(c *fiber.Ctx) error { } if proxyMode { - return proxyHandler(c, reqURI) + return proxyHandler(c, reqURIwithQuery) } // Check the original image for existence, @@ -64,8 +90,11 @@ func convert(c *fiber.Ctx) error { } // generate with timestamp to make sure files are update-to-date - avifAbs, webpAbs := genOptimizedAbsPath(rawImageAbs, config.ExhaustPath, imgFilename, reqURI) - convertFilter(rawImageAbs, avifAbs, webpAbs, nil) + // If extraParams not enabled, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp + // If extraParams enabled, and given request at tsuki.jpg?width=200, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=200&height=0 + // If extraParams enabled, and given request at tsuki.jpg, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=0&height=0 + avifAbs, webpAbs := genOptimizedAbsPath(rawImageAbs, config.ExhaustPath, imgFilename, reqURI, extraParams) + convertFilter(rawImageAbs, avifAbs, webpAbs, extraParams, nil) var availableFiles = []string{rawImageAbs} for _, v := range goodFormat { @@ -91,25 +120,29 @@ func convert(c *fiber.Ctx) error { return c.SendFile(finalFileName) } -func proxyHandler(c *fiber.Ctx, reqURI string) error { - // https://test.webp.sh/node.png - realRemoteAddr := config.ImgPath + reqURI +func proxyHandler(c *fiber.Ctx, reqURIwithQuery string) error { + // https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200 + realRemoteAddr := config.ImgPath + reqURIwithQuery + + // Since we cannot store file in format of "mypic/123.jpg?someother=200&somebugs=200", we need to hash it. + reqURIwithQueryHash := Sha1Path(reqURIwithQuery) // 378e740ca56144b7587f3af9debeee544842879a + + localRawImagePath := remoteRaw + "/" + reqURIwithQueryHash // For store the remote raw image, /home/webp_server/remote-raw/378e740ca56144b7587f3af9debeee544842879a // Ping Remote for status code and etag info log.Infof("Remote Addr is %s fetching", realRemoteAddr) statusCode, etagValue, remoteLength := getRemoteImageInfo(realRemoteAddr) + localEtagWebPPath := config.ExhaustPath + "/" + reqURIwithQueryHash + "-etag-" + etagValue // For store the remote webp image, /home/webp_server/exhaust/378e740ca56144b7587f3af9debeee544842879a-etag- + if statusCode == 200 { - // Check local path: /node.png-etag- - localEtagWebPPath := config.ExhaustPath + reqURI + "-etag-" + etagValue if imageExists(localEtagWebPPath) { chooseProxy(remoteLength, localEtagWebPPath) return c.SendFile(localEtagWebPPath) } else { // Temporary store of remote file. - cleanProxyCache(config.ExhaustPath + reqURI + "*") - localRawImagePath := remoteRaw + reqURI + cleanProxyCache(config.ExhaustPath + reqURIwithQuery + "*") _ = fetchRemoteImage(localRawImagePath, realRemoteAddr) _ = os.MkdirAll(path.Dir(localEtagWebPPath), 0755) - encodeErr := webpEncoder(localRawImagePath, localEtagWebPPath, config.Quality) + encodeErr := webpEncoder(localRawImagePath, localEtagWebPPath, config.Quality, ExtraParams{Width: 0, Height: 0}) if encodeErr != nil { // Send as it is. return c.SendFile(localRawImagePath) @@ -122,7 +155,7 @@ func proxyHandler(c *fiber.Ctx, reqURI string) error { _ = c.Send([]byte(msg)) log.Warn(msg) _ = c.SendStatus(statusCode) - cleanProxyCache(config.ExhaustPath + reqURI + "*") + cleanProxyCache(config.ExhaustPath + reqURIwithQuery + "*") return errors.New(msg) } }