Support extra parameters on request (#192)

* version 1

* some update

* Fix ST1020

* Remove ST1020

* Fix CI

* Fix params

* Fix order

* Allow reqURIwithQuery pass through when proxy mode

* Fix bugs

* Extract resizeImage

* Check for err

config.go

@@ -1,5 +1,7 @@
 package main
 
+import "fmt"
+
 type Config struct {
 	Host              string   `json:"HOST"`
 	Port              string   `json:"PORT"`
@@ -8,6 +10,17 @@ type Config struct {
 	AllowedTypes      []string `json:"ALLOWED_TYPES"`
 	ExhaustPath       string   `json:"EXHAUST_PATH"`
 	EnableAVIF        bool     `json:"ENABLE_AVIF"`
+	EnableExtraParams bool     `json:"ENABLE_EXTRA_PARAMS"`
+}
+
+type ExtraParams struct {
+	Width  int // in px
+	Height int // in px
+}
+
+// String : convert ExtraParams to string, used to generate cache path
+func (e *ExtraParams) String() string {
+	return fmt.Sprintf("_width=%d&height=%d", e.Width, e.Height)
 }
 
 var (
@@ -18,7 +31,7 @@ var (
 	prefetch, proxyMode      bool
 	remoteRaw                = "remote-raw"
 	config                   Config
-	version                  = "0.6.0"
+	version                  = "0.7.0"
 )
 
 const (
@@ -30,7 +43,8 @@ const (
   "IMG_PATH": "./pics",
   "EXHAUST_PATH": "./exhaust",
   "ALLOWED_TYPES": ["jpg","png","jpeg","bmp"],
-  "ENABLE_AVIF": false
+  "ENABLE_AVIF": false,
+  "ENABLE_EXTRA_PARAMS": false
 }`
 
 	sampleSystemd = `

config.json

@@ -5,5 +5,6 @@
   "IMG_PATH": "./pics",
   "EXHAUST_PATH": "./exhaust",
   "ALLOWED_TYPES": ["jpg","png","jpeg","bmp"],
-  "ENABLE_AVIF": false
+  "ENABLE_AVIF": false,
+  "ENABLE_EXTRA_PARAMS": false
 }

encoder.go

@@ -12,14 +12,35 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-func convertFilter(raw, avifPath, webpPath string, c chan int) {
+func resizeImage(img *vips.ImageRef, extraParams ExtraParams) error {
+	imgHeightWidthRatio := float32(img.Metadata().Height) / float32(img.Metadata().Width)
+	if extraParams.Width > 0 && extraParams.Height > 0 {
+		err := img.Thumbnail(extraParams.Width, extraParams.Height, 0)
+		if err != nil {
+			return err
+		}
+	} else if extraParams.Width > 0 && extraParams.Height == 0 {
+		err := img.Thumbnail(extraParams.Width, int(float32(extraParams.Width)*imgHeightWidthRatio), 0)
+		if err != nil {
+			return err
+		}
+	} else if extraParams.Height > 0 && extraParams.Width == 0 {
+		err := img.Thumbnail(int(float32(extraParams.Height)/imgHeightWidthRatio), extraParams.Height, 0)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func convertFilter(raw, avifPath, webpPath string, extraParams ExtraParams, c chan int) {
 	// all absolute paths
 
 	var wg sync.WaitGroup
 	wg.Add(2)
 	if !imageExists(avifPath) && config.EnableAVIF {
 		go func() {
-			err := convertImage(raw, avifPath, "avif")
+			err := convertImage(raw, avifPath, "avif", extraParams)
 			if err != nil {
 				log.Errorln(err)
 			}
@@ -31,7 +52,7 @@ func convertFilter(raw, avifPath, webpPath string, c chan int) {
 
 	if !imageExists(webpPath) {
 		go func() {
-			err := convertImage(raw, webpPath, "webp")
+			err := convertImage(raw, webpPath, "webp", extraParams)
 			if err != nil {
 				log.Errorln(err)
 			}
@@ -47,11 +68,12 @@ func convertFilter(raw, avifPath, webpPath string, c chan int) {
 	}
 }
 
-func convertImage(raw, optimized, itype string) error {
+func convertImage(raw, optimized, itype string, extraParams ExtraParams) error {
-	// we don't have abc.jpg.png1582558990.webp
+	// we don't have /path/to/tsuki.jpg.1582558990.webp, maybe we have /path/to/tsuki.jpg.1082008000.webp
-	// delete the old pic and convert a new one.
+	// delete the old converted pic and convert a new one.
 	// optimized: /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
 	// we'll delete file starts with /home/webp_server/exhaust/path/to/tsuki.jpg.ts.itype
+	// If contain extraParams like tsuki.jpg?width=200, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp?width=200
 
 	s := strings.Split(path.Base(optimized), ".")
 	pattern := path.Join(path.Dir(optimized), s[0]+"."+s[1]+".*."+s[len(s)-1])
@@ -73,24 +95,33 @@ func convertImage(raw, optimized, itype string) error {
 
 	switch itype {
 	case "webp":
-		err = webpEncoder(raw, optimized, config.Quality)
+		err = webpEncoder(raw, optimized, config.Quality, extraParams)
 	case "avif":
-		err = avifEncoder(raw, optimized, config.Quality)
+		err = avifEncoder(raw, optimized, config.Quality, extraParams)
 	}
 	return err
 }
 
-func avifEncoder(p1, p2 string, quality int) error {
+func avifEncoder(p1, p2 string, quality int, extraParams ExtraParams) error {
 	// if convert fails, return error; success nil
 	var buf []byte
 	img, err := vips.NewImageFromFile(p1)
 	if err != nil {
 		return err
 	}
+
+	if config.EnableExtraParams {
+		err = resizeImage(img, extraParams)
+		if err != nil {
+			return err
+		}
+	}
+
 	// AVIF has a maximum resolution of 65536 x 65536 pixels.
 	if img.Metadata().Width > avifMax || img.Metadata().Height > avifMax {
-		return errors.New("WebP: image too large")
+		return errors.New("AVIF: image too large")
 	}
+
 	// If quality >= 100, we use lossless mode
 	if quality >= 100 {
 		buf, _, err = img.ExportAvif(&vips.AvifExportParams{
@@ -118,7 +149,7 @@ func avifEncoder(p1, p2 string, quality int) error {
 	return nil
 }
 
-func webpEncoder(p1, p2 string, quality int) error {
+func webpEncoder(p1, p2 string, quality int, extraParams ExtraParams) error {
 	// if convert fails, return error; success nil
 	var buf []byte
 	img, err := vips.NewImageFromFile(p1)
@@ -126,10 +157,18 @@ func webpEncoder(p1, p2 string, quality int) error {
 		return err
 	}
 
+	if config.EnableExtraParams {
+		err = resizeImage(img, extraParams)
+		if err != nil {
+			return err
+		}
+	}
+
 	// The maximum pixel dimensions of a WebP image is 16383 x 16383.
 	if img.Metadata().Width > webpMax || img.Metadata().Height > webpMax {
 		return errors.New("WebP: image too large")
 	}
+
 	// If quality >= 100, we use lossless mode
 	if quality >= 100 {
 		buf, _, err = img.ExportWebp(&vips.WebpExportParams{

encoder_test.go

@@ -34,25 +34,25 @@ func TestWebPEncoder(t *testing.T) {
 
 func TestAvifEncoder(t *testing.T) {
 	// Only one file: img_over_16383px.jpg might cause memory issues on CI environment
-	assert.Nil(t, avifEncoder("./pics/big.jpg", dest, 80))
+	assert.Nil(t, avifEncoder("./pics/big.jpg", dest, 80, ExtraParams{Width: 0, Height: 0}))
 	assertType(t, dest, "image/avif")
 }
 
 func TestNonExistImage(t *testing.T) {
-	assert.NotNil(t, webpEncoder("./pics/empty.jpg", dest, 80))
+	assert.NotNil(t, webpEncoder("./pics/empty.jpg", dest, 80, ExtraParams{Width: 0, Height: 0}))
-	assert.NotNil(t, avifEncoder("./pics/empty.jpg", dest, 80))
+	assert.NotNil(t, avifEncoder("./pics/empty.jpg", dest, 80, ExtraParams{Width: 0, Height: 0}))
 }
 
 func TestHighResolutionImage(t *testing.T) {
-	assert.NotNil(t, webpEncoder("./pics/img_over_16383px.jpg", dest, 80))
+	assert.NotNil(t, webpEncoder("./pics/img_over_16383px.jpg", dest, 80, ExtraParams{Width: 0, Height: 0}))
-	assert.Nil(t, avifEncoder("./pics/img_over_16383px.jpg", dest, 80))
+	assert.Nil(t, avifEncoder("./pics/img_over_16383px.jpg", dest, 80, ExtraParams{Width: 0, Height: 0}))
 }
 
 func runEncoder(t *testing.T, file string, dest string) {
 	if file == "pics/empty.jpg" {
 		t.Log("Empty file, that's okay.")
 	}
-	_ = webpEncoder(file, dest, 80)
+	_ = webpEncoder(file, dest, 80, ExtraParams{Width: 0, Height: 0})
 	assertType(t, dest, "image/webp")
 
 }

helper.go

@@ -2,6 +2,8 @@ package main
 
 import (
 	"bytes"
+	"crypto/sha1" //#nosec
+	"encoding/hex"
 	"fmt"
 	"hash/crc32"
 	"io"
@@ -136,7 +138,7 @@ func cleanProxyCache(cacheImagePath string) {
 	}
 }
 
-func genOptimizedAbsPath(rawImagePath string, exhaustPath string, imageName string, reqURI string) (string, string) {
+func genOptimizedAbsPath(rawImagePath string, exhaustPath string, imageName string, reqURI string, extraParams ExtraParams) (string, string) {
 	// get file mod time
 	STAT, err := os.Stat(rawImagePath)
 	if err != nil {
@@ -149,6 +151,14 @@ func genOptimizedAbsPath(rawImagePath string, exhaustPath string, imageName stri
 	// avifFilename: abc.jpg.png -> abc.jpg.png.1582558990.avif
 	avifFilename := fmt.Sprintf("%s.%d.avif", imageName, ModifiedTime)
 
+	// If extraParams not enabled, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
+	// If extraParams enabled, and given request at tsuki.jpg?width=200, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=200&height=0
+	// If extraParams enabled, and given request at tsuki.jpg, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=0&height=0
+	if config.EnableExtraParams {
+		webpFilename = webpFilename + extraParams.String()
+		avifFilename = avifFilename + extraParams.String()
+	}
+
 	// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
 	// Custom Exhaust: /path/to/exhaust/web_path/web_to/tsuki.jpg.1582558990.webp
 	webpAbsolutePath := path.Clean(path.Join(exhaustPath, path.Dir(reqURI), webpFilename))
@@ -216,7 +226,6 @@ func guessSupportedFormat(header *fasthttp.RequestHeader) []string {
 		}
 	}
 	return accepted
-
 }
 
 func chooseProxy(proxyRawSize string, optimizedAbs string) bool {
@@ -246,3 +255,10 @@ func findSmallestFiles(files []string) string {
 	}
 	return final
 }
+
+func Sha1Path(uri string) string {
+	/* #nosec */
+	h := sha1.New()
+	h.Write([]byte(uri))
+	return hex.EncodeToString(h.Sum(nil))
+}

helper_test.go

@@ -47,7 +47,7 @@ func TestImageExists(t *testing.T) {
 
 func TestGenWebpAbs(t *testing.T) {
 	cwd, cooked := genOptimizedAbsPath("./pics/webp_server.png", "/tmp",
-		"test", "a")
+		"test", "a", ExtraParams{Width: 0, Height: 0})
 	if !strings.Contains(cwd, "webp_server_go") {
 		t.Logf("Result: [%v], Expected: [%v]", cwd, "webp_server_go")
 	}

prefetch.go

@@ -34,10 +34,10 @@ func prefetchImages(confImgPath string, ExhaustPath string) {
 			}
 			// RawImagePath string, ImgFilename string, reqURI string
 			proposedURI := strings.Replace(picAbsPath, confImgPath, "", 1)
-			avif, webp := genOptimizedAbsPath(picAbsPath, ExhaustPath, info.Name(), proposedURI)
+			avif, webp := genOptimizedAbsPath(picAbsPath, ExhaustPath, info.Name(), proposedURI, ExtraParams{Width: 0, Height: 0})
 			_ = os.MkdirAll(path.Dir(avif), 0755)
 			log.Infof("Prefetching %s", picAbsPath)
-			go convertFilter(picAbsPath, avif, webp, finishChan)
+			go convertFilter(picAbsPath, avif, webp, ExtraParams{Width: 0, Height: 0}, finishChan)
 			_ = bar.Add(<-finishChan)
 			return nil
 		})

router.go

@@ -7,6 +7,7 @@ import (
 	"net/url"
 	"os"
 	"path"
+	"strconv"
 
 	"github.com/gofiber/fiber/v2"
 	log "github.com/sirupsen/logrus"
@@ -15,13 +16,38 @@ import (
 func convert(c *fiber.Ctx) error {
 	//basic vars
 	var reqURI, _ = url.QueryUnescape(c.Path())                 // /mypic/123.jpg
-
+	var reqURIwithQuery, _ = url.QueryUnescape(c.OriginalURL()) // /mypic/123.jpg?someother=200&somebugs=200
+	// Sometimes reqURIwithQuery can be https://example.tld/mypic/123.jpg?someother=200&somebugs=200, we need to extract it.
+	u, err := url.Parse(reqURIwithQuery)
+	if err != nil {
+		log.Errorln(err)
+	}
+	reqURIwithQuery = u.RequestURI()
 	// delete ../ in reqURI to mitigate directory traversal
 	reqURI = path.Clean(reqURI)
+	reqURIwithQuery = path.Clean(reqURIwithQuery)
+
+	// Begin Extra params
+	var extraParams ExtraParams
+	Width := c.Query("width")
+	Height := c.Query("height")
+	WidthInt, err := strconv.Atoi(Width)
+	if err != nil {
+		WidthInt = 0
+	}
+	HeightInt, err := strconv.Atoi(Height)
+	if err != nil {
+		HeightInt = 0
+	}
+	extraParams = ExtraParams{
+		Width:  WidthInt,
+		Height: HeightInt,
+	}
+	// End Extra params
 
 	var rawImageAbs string
 	if proxyMode {
-		rawImageAbs = config.ImgPath + reqURI
+		rawImageAbs = config.ImgPath + reqURIwithQuery // https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200
 	} else {
 		rawImageAbs = path.Join(config.ImgPath, reqURI) // /home/xxx/mypic/123.jpg
 	}
@@ -51,7 +77,7 @@ func convert(c *fiber.Ctx) error {
 	}
 
 	if proxyMode {
-		return proxyHandler(c, reqURI)
+		return proxyHandler(c, reqURIwithQuery)
 	}
 
 	// Check the original image for existence,
@@ -64,8 +90,11 @@ func convert(c *fiber.Ctx) error {
 	}
 
 	// generate with timestamp to make sure files are update-to-date
-	avifAbs, webpAbs := genOptimizedAbsPath(rawImageAbs, config.ExhaustPath, imgFilename, reqURI)
+	// If extraParams not enabled, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
-	convertFilter(rawImageAbs, avifAbs, webpAbs, nil)
+	// If extraParams enabled, and given request at tsuki.jpg?width=200, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=200&height=0
+	// If extraParams enabled, and given request at tsuki.jpg, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=0&height=0
+	avifAbs, webpAbs := genOptimizedAbsPath(rawImageAbs, config.ExhaustPath, imgFilename, reqURI, extraParams)
+	convertFilter(rawImageAbs, avifAbs, webpAbs, extraParams, nil)
 
 	var availableFiles = []string{rawImageAbs}
 	for _, v := range goodFormat {
@@ -91,25 +120,29 @@ func convert(c *fiber.Ctx) error {
 	return c.SendFile(finalFileName)
 }
 
-func proxyHandler(c *fiber.Ctx, reqURI string) error {
+func proxyHandler(c *fiber.Ctx, reqURIwithQuery string) error {
-	// https://test.webp.sh/node.png
+	// https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200
-	realRemoteAddr := config.ImgPath + reqURI
+	realRemoteAddr := config.ImgPath + reqURIwithQuery
+
+	// Since we cannot store file in format of "mypic/123.jpg?someother=200&somebugs=200", we need to hash it.
+	reqURIwithQueryHash := Sha1Path(reqURIwithQuery) // 378e740ca56144b7587f3af9debeee544842879a
+
+	localRawImagePath := remoteRaw + "/" + reqURIwithQueryHash // For store the remote raw image, /home/webp_server/remote-raw/378e740ca56144b7587f3af9debeee544842879a
 	// Ping Remote for status code and etag info
 	log.Infof("Remote Addr is %s fetching", realRemoteAddr)
 	statusCode, etagValue, remoteLength := getRemoteImageInfo(realRemoteAddr)
+	localEtagWebPPath := config.ExhaustPath + "/" + reqURIwithQueryHash + "-etag-" + etagValue // For store the remote webp image, /home/webp_server/exhaust/378e740ca56144b7587f3af9debeee544842879a-etag-<etagValue>
+
 	if statusCode == 200 {
-		// Check local path: /node.png-etag-<etagValue>
-		localEtagWebPPath := config.ExhaustPath + reqURI + "-etag-" + etagValue
 		if imageExists(localEtagWebPPath) {
 			chooseProxy(remoteLength, localEtagWebPPath)
 			return c.SendFile(localEtagWebPPath)
 		} else {
 			// Temporary store of remote file.
-			cleanProxyCache(config.ExhaustPath + reqURI + "*")
+			cleanProxyCache(config.ExhaustPath + reqURIwithQuery + "*")
-			localRawImagePath := remoteRaw + reqURI
 			_ = fetchRemoteImage(localRawImagePath, realRemoteAddr)
 			_ = os.MkdirAll(path.Dir(localEtagWebPPath), 0755)
-			encodeErr := webpEncoder(localRawImagePath, localEtagWebPPath, config.Quality)
+			encodeErr := webpEncoder(localRawImagePath, localEtagWebPPath, config.Quality, ExtraParams{Width: 0, Height: 0})
 			if encodeErr != nil {
 				// Send as it is.
 				return c.SendFile(localRawImagePath)
@@ -122,7 +155,7 @@ func proxyHandler(c *fiber.Ctx, reqURI string) error {
 		_ = c.Send([]byte(msg))
 		log.Warn(msg)
 		_ = c.SendStatus(statusCode)
-		cleanProxyCache(config.ExhaustPath + reqURI + "*")
+		cleanProxyCache(config.ExhaustPath + reqURIwithQuery + "*")
 		return errors.New(msg)
 	}
 }