Multiple backends support (#207)

* Fix: h2non/filetype upgraded to support avif signatures

* Fix: make clean updated to include test/output dirs

* Feature: multi-backend support via IMG_MAP config key as described in #217

* feat: implement both local and remote (proxyMode) mappings for multi-backend

* Feature: multi-backend support via IMG_MAP config key as described in #217

* fix: go-is-svg should be direct import

* fix: imgMap paths are relative to CWD

* feature: IMG_MAP is parsed on start

---------

Co-authored-by: Nova Kwok <n0vad3v@riseup.net>

Makefile

@@ -37,7 +37,7 @@ test:
 	go test -v -coverprofile=coverage.txt -covermode=atomic ./...
 
 clean:
-	rm -rf prefetch remote-raw exhaust tools coverage.txt
+	rm -rf prefetch remote-raw exhaust tools coverage.txt metadata exhaust_test
 
 
 docker:

config.json

@@ -5,6 +5,7 @@
   "IMG_PATH": "./pics",
   "EXHAUST_PATH": "./exhaust",
   "ALLOWED_TYPES": ["jpg","png","jpeg","bmp","gif","svg"],
+  "IMG_MAP": {},
   "ENABLE_AVIF": false,
   "ENABLE_EXTRA_PARAMS": false
 }

config/config.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"regexp"
 	"runtime"
+	"strings"
 	"time"
 
 	"github.com/patrickmn/go-cache"
@@ -17,8 +18,7 @@ const (
 	FiberLogFormat = "${ip} - [${time}] ${method} ${url} ${status} ${referer} ${ua}\n"
 	WebpMax        = 16383
 	AvifMax        = 65536
-	RemoteRaw      = "remote-raw"
+	HttpRegexp     = `^https?://`
-
 	SampleConfig   = `
 {
   "HOST": "127.0.0.1",
@@ -26,6 +26,7 @@ const (
   "QUALITY": "80",
   "IMG_PATH": "./pics",
   "EXHAUST_PATH": "./exhaust",
+  "IMG_MAP": {},
   "ALLOWED_TYPES": ["jpg","png","jpeg","bmp","svg"],
   "ENABLE_AVIF": false,
   "ENABLE_EXTRA_PARAMS": false
@@ -60,10 +61,11 @@ var (
 	Config      jsonFile
 	Version     = "0.9.8"
 	WriteLock   = cache.New(5*time.Minute, 10*time.Minute)
+	RemoteRaw   = "./remote-raw"
+	Metadata    = "./metadata"
+	LocalHostAlias = "local"
 )
 
-const Metadata = "metadata"
-
 type MetaFile struct {
 	Id       string `json:"id"`       // hash of below path️, also json file name id.webp
 	Path     string `json:"path"`     // local: path with width and height, proxy: full url
@@ -76,6 +78,7 @@ type jsonFile struct {
 	ImgPath           string            `json:"IMG_PATH"`
 	Quality           int               `json:"QUALITY,string"`
 	AllowedTypes      []string          `json:"ALLOWED_TYPES"`
+	ImageMap          map[string]string `json:"IMG_MAP"`
 	ExhaustPath       string            `json:"EXHAUST_PATH"`
 	EnableAVIF        bool              `json:"ENABLE_AVIF"`
 	EnableExtraParams bool              `json:"ENABLE_EXTRA_PARAMS"`
@@ -99,6 +102,22 @@ func LoadConfig() {
 	_ = decoder.Decode(&Config)
 	_ = jsonObject.Close()
 	switchProxyMode()
+	Config.ImageMap = parseImgMap(Config.ImageMap)
+}
+
+func parseImgMap(imgMap map[string]string) map[string]string {
+	var parsedImgMap = map[string]string{}
+	httpRegexpMatcher := regexp.MustCompile(HttpRegexp)
+	for uriMap, uriMapTarget := range imgMap {
+		if httpRegexpMatcher.Match([]byte(uriMap)) || strings.HasPrefix(uriMap, "/") {
+			// Valid
+			parsedImgMap[uriMap] = uriMapTarget
+		} else {
+			// Invalid
+			log.Warnf("IMG_MAP key '%s' does matches '%s' or starts with '/' - skipped", uriMap, HttpRegexp)
+		}
+	}
+	return parsedImgMap
 }
 
 type ExtraParams struct {
@@ -107,8 +126,10 @@ type ExtraParams struct {
 }
 
 func switchProxyMode() {
-	matched, _ := regexp.MatchString(`^https?://`, Config.ImgPath)
+	matched, _ := regexp.MatchString(HttpRegexp, Config.ImgPath)
 	if matched {
+		// Enable proxy based on ImgPath should be deprecated in future versions
+		log.Warn("Enable proxy based on ImgPath will be deprecated in future versions. Use IMG_MAP config options instead")
 		ProxyMode = true
 	}
 }

config/config_test.go

@@ -19,6 +19,7 @@ func TestLoadConfig(t *testing.T) {
 	assert.Equal(t, Config.Port, "3333")
 	assert.Equal(t, Config.Quality, 80)
 	assert.Equal(t, Config.ImgPath, "./pics")
+	assert.Equal(t, Config.ImageMap, map[string]string{})
 	assert.Equal(t, Config.ExhaustPath, "./exhaust")
 }
 
@@ -29,3 +30,26 @@ func TestSwitchProxyMode(t *testing.T) {
 	switchProxyMode()
 	assert.True(t, ProxyMode)
 }
+
+func TestParseImgMap(t *testing.T) {
+	empty := map[string]string{}
+	good := map[string]string{
+		"/1": "../pics/dir1",
+		"http://example.com": "../pics",
+		"https://example.com": "../pics",
+	}
+	bad := map[string]string{
+		"1": "../pics/dir1",
+		"httpx://example.com": "../pics",
+		"ftp://example.com": "../pics",
+	}
+	
+	assert.Equal(t, empty, parseImgMap(empty))
+	assert.Equal(t, empty, parseImgMap(bad))
+	assert.Equal(t, good, parseImgMap(good))
+
+	for k, v := range good {
+		bad[k] = v
+	}
+	assert.Equal(t, good, parseImgMap(bad))
+}

encoder/encoder.go

@@ -86,9 +86,20 @@ func ConvertFilter(raw, avifPath, webpPath string, extraParams config.ExtraParam
 
 func ResizeItself(raw, dest string, extraParams config.ExtraParams) {
 	log.Infof("Resize %s itself to %s", raw, dest)
-	img, _ := vips.LoadImageFromFile(raw, &vips.ImportParams{
+
+	// we need to create dir first
+	var err = os.MkdirAll(path.Dir(dest), 0755)
+	if err != nil {
+		log.Error(err.Error())
+	}
+
+	img, err := vips.LoadImageFromFile(raw, &vips.ImportParams{
 		FailOnError: boolFalse,
 	})
+	if err != nil {
+		log.Warnf("Could not load %s: %s", raw, err)
+		return
+	}
 	_ = resizeImage(img, extraParams)
 	buf, _, _ := img.ExportNative()
 	_ = os.WriteFile(dest, buf, 0600)

encoder/prefetch.go

@@ -34,8 +34,8 @@ func PrefetchImages() {
 				return nil
 			}
 			// RawImagePath string, ImgFilename string, reqURI string
-			metadata := helper.ReadMetadata(picAbsPath, "")
+			metadata := helper.ReadMetadata(picAbsPath, "", config.LocalHostAlias)
-			avif, webp := helper.GenOptimizedAbsPath(metadata)
+			avif, webp := helper.GenOptimizedAbsPath(metadata, config.LocalHostAlias)
 			_ = os.MkdirAll(path.Dir(avif), 0755)
 			log.Infof("Prefetching %s", picAbsPath)
 			go ConvertFilter(picAbsPath, avif, webp, config.ExtraParams{Width: 0, Height: 0}, finishChan)

go.mod

@@ -6,7 +6,8 @@ require (
 	github.com/cespare/xxhash v1.1.0
 	github.com/davidbyttow/govips/v2 v2.13.0
 	github.com/gofiber/fiber/v2 v2.48.0
-	github.com/h2non/filetype v1.1.3
+	github.com/h2non/filetype v1.1.4-0.20230123234534-cfcd7d097bc4
+	github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/schollz/progressbar/v3 v3.13.1
 	github.com/sirupsen/logrus v1.9.3
@@ -18,7 +19,6 @@ require (
 	github.com/andybalholm/brotli v1.0.5 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/google/uuid v1.3.0 // indirect
-	github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c // indirect
 	github.com/klauspost/compress v1.16.3 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect

go.sum

@@ -13,8 +13,8 @@ github.com/gofiber/fiber/v2 v2.48.0 h1:cRVMCb9aUJDsyHxGFLwz/sGzDggdailZZyptU9F9c
 github.com/gofiber/fiber/v2 v2.48.0/go.mod h1:xqJgfqrc23FJuqGOW6DVgi3HyZEm2Mn9pRqUb2kHSX8=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
+github.com/h2non/filetype v1.1.4-0.20230123234534-cfcd7d097bc4 h1:k7FGP5I7raiaC3aAzCLddcoxzboIrOm6/FVRXjp/5JM=
-github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
+github.com/h2non/filetype v1.1.4-0.20230123234534-cfcd7d097bc4/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
 github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c h1:fEE5/5VNnYUoBOj2I9TP8Jc+a7lge3QWn9DKE7NCwfc=
 github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c/go.mod h1:ObS/W+h8RYb1Y7fYivughjxojTmIu5iAIjSrSLCLeqE=
 github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=

handler/remote.go

@@ -75,18 +75,18 @@ func downloadFile(filepath string, url string) {
 
 }
 
-func fetchRemoteImg(url string) config.MetaFile {
+func fetchRemoteImg(url string, subdir string) config.MetaFile {
 	// url is https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200
 	// How do we know if the remote img is changed? we're using hash(etag+length)
 	log.Infof("Remote Addr is %s, pinging for info...", url)
 	etag := pingURL(url)
-	metadata := helper.ReadMetadata(url, etag)
+	metadata := helper.ReadMetadata(url, etag, subdir)
-	localRawImagePath := path.Join(config.RemoteRaw, metadata.Id)
+	localRawImagePath := path.Join(config.RemoteRaw, subdir, metadata.Id)
 
 	if !helper.ImageExists(localRawImagePath) || metadata.Checksum != helper.HashString(etag) {
 		// remote file has changed or local file not exists
 		log.Info("Remote file not found in remote-raw, re-fetching...")
-		cleanProxyCache(path.Join(config.Config.ExhaustPath, metadata.Id+"*"))
+		cleanProxyCache(path.Join(config.Config.ExhaustPath, subdir, metadata.Id+"*"))
 		downloadFile(localRawImagePath, url)
 	}
 	return metadata

handler/router.go

@@ -3,6 +3,8 @@ package handler
 import (
 	"net/http"
 	"net/url"
+	"regexp"
+	"strings"
 	"webp_server_go/config"
 	"webp_server_go/encoder"
 	"webp_server_go/helper"
@@ -21,11 +23,20 @@ func Convert(c *fiber.Ctx) error {
 	// 3. pass it to encoder, get the result, send it back
 
 	var (
+		reqHostname        = c.Hostname()
+		reqHost            = c.Protocol() + "://" + reqHostname // http://www.example.com:8000
 		reqURI, _          = url.QueryUnescape(c.Path())        // /mypic/123.jpg
 		reqURIwithQuery, _ = url.QueryUnescape(c.OriginalURL()) // /mypic/123.jpg?someother=200&somebugs=200
 		filename           = path.Base(reqURI)
+		realRemoteAddr     = ""
+		targetHostName     = config.LocalHostAlias
+		targetHost         = config.Config.ImgPath
+		proxyMode          = config.ProxyMode
+		mapMode            = false
 	)
 
+	log.Debugf("Incoming connection from %s %s %s", c.IP(), reqHostname, reqURIwithQuery)
+
 	if !helper.CheckAllowedType(filename) {
 		msg := "File extension not allowed! " + filename
 		log.Warn(msg)
@@ -47,29 +58,84 @@ func Convert(c *fiber.Ctx) error {
 		Height: height,
 	}
 
+	// Rewrite the target backend if a mapping rule matches the hostname
+	if hostMap, hostMapFound := config.Config.ImageMap[reqHost]; hostMapFound {
+		log.Debugf("Found host mapping %s -> %s", reqHostname, hostMap)
+		targetHostUrl, _ := url.Parse(hostMap)
+		targetHostName = targetHostUrl.Host
+		targetHost = targetHostUrl.Scheme + "://" + targetHostUrl.Host
+		proxyMode = true
+	} else {
+		// There's not matching host mapping, now check for any URI map that apply
+		httpRegexpMatcher := regexp.MustCompile(config.HttpRegexp)
+		for uriMap, uriMapTarget := range config.Config.ImageMap {
+			if strings.HasPrefix(reqURI, uriMap) {
+				log.Debugf("Found URI mapping %s -> %s", uriMap, uriMapTarget)
+				mapMode = true
+
+				// if uriMapTarget we use the proxy mode to fetch the remote
+				if httpRegexpMatcher.Match([]byte(uriMapTarget)) {
+					targetHostUrl, _ := url.Parse(uriMapTarget)
+					targetHostName = targetHostUrl.Host
+					targetHost = targetHostUrl.Scheme + "://" + targetHostUrl.Host
+					reqURI          = strings.Replace(reqURI,          uriMap, targetHostUrl.Path, 1)
+					reqURIwithQuery = strings.Replace(reqURIwithQuery, uriMap, targetHostUrl.Path, 1)
+					proxyMode = true
+				} else {
+					reqURI          = strings.Replace(reqURI,          uriMap, uriMapTarget, 1)
+					reqURIwithQuery = strings.Replace(reqURIwithQuery, uriMap, uriMapTarget, 1)
+				}
+				break
+			}
+		}
+
+	}
+
+	if proxyMode {
+
+		if !mapMode {
+			// Don't deal with the encoding to avoid upstream compatibilities
+			reqURI = c.Path()
+			reqURIwithQuery = c.OriginalURL()
+		}
+
+		log.Tracef("reqURIwithQuery is %s", reqURIwithQuery)
+
+		// Replace host in the URL
+		// realRemoteAddr = strings.Replace(reqURIwithQuery, reqHost, targetHost, 1)
+		realRemoteAddr = targetHost + reqURIwithQuery
+		log.Debugf("realRemoteAddr is %s", realRemoteAddr)
+	}
+	
 	var rawImageAbs string
 	var metadata = config.MetaFile{}
-	if config.ProxyMode {
+	if proxyMode {
 		// this is proxyMode, we'll have to use this url to download and save it to local path, which also gives us rawImageAbs
 		// https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200
-		metadata = fetchRemoteImg(config.Config.ImgPath + reqURIwithQuery)
+		
-		rawImageAbs = path.Join(config.RemoteRaw, metadata.Id)
+		metadata = fetchRemoteImg(realRemoteAddr, targetHostName)
+		rawImageAbs = path.Join(config.RemoteRaw, targetHostName,  metadata.Id)
 	} else {
 		// not proxyMode, we'll use local path
-		metadata = helper.ReadMetadata(reqURIwithQuery, "")
+		metadata = helper.ReadMetadata(reqURIwithQuery, "", targetHostName)
+		if !mapMode {
+			// by default images are hosted in ImgPath
 			rawImageAbs = path.Join(config.Config.ImgPath, reqURI)
+		} else {
+			rawImageAbs = reqURI
+		}
 		// detect if source file has changed
 		if metadata.Checksum != helper.HashFile(rawImageAbs) {
 			log.Info("Source file has changed, re-encoding...")
-			helper.WriteMetadata(reqURIwithQuery, "")
+			helper.WriteMetadata(reqURIwithQuery, "", targetHostName)
-			cleanProxyCache(path.Join(config.Config.ExhaustPath, metadata.Id))
+			cleanProxyCache(path.Join(config.Config.ExhaustPath, targetHostName, metadata.Id))
 		}
 	}
 
 	goodFormat := helper.GuessSupportedFormat(&c.Request().Header)
 	// resize itself and return if only one format(raw) is supported
 	if len(goodFormat) == 1 {
-		dest := path.Join(config.Config.ExhaustPath, metadata.Id)
+		dest := path.Join(config.Config.ExhaustPath, targetHostName, metadata.Id)
 		if !helper.ImageExists(dest) {
 			encoder.ResizeItself(rawImageAbs, dest, extraParams)
 		}
@@ -85,7 +151,7 @@ func Convert(c *fiber.Ctx) error {
 		return nil
 	}
 
-	avifAbs, webpAbs := helper.GenOptimizedAbsPath(metadata)
+	avifAbs, webpAbs := helper.GenOptimizedAbsPath(metadata, targetHostName)
 	encoder.ConvertFilter(rawImageAbs, avifAbs, webpAbs, extraParams, nil)
 
 	var availableFiles = []string{rawImageAbs}

handler/router_test.go

@@ -0,0 +1,341 @@
+package handler
+
+import (
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"testing"
+	"webp_server_go/config"
+	"webp_server_go/helper"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/gofiber/fiber/v2/middleware/etag"
+	"github.com/stretchr/testify/assert"
+)
+
+var (
+	chromeUA     = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36"
+	acceptWebP   = "image/webp,image/apng,image/*,*/*;q=0.8"
+	acceptAvif   = "image/avif,image/*,*/*;q=0.8"
+	acceptLegacy = "image/jpeg"
+	safariUA     = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1 Safari/605.1.15"
+	curlUA       = "curl/7.64.1"
+)
+
+func setupParam() {
+	// setup parameters here...
+	config.Config.ImgPath = "../pics"
+	config.Config.ExhaustPath = "../exhaust_test"
+	config.Config.AllowedTypes = []string{"jpg", "png", "jpeg", "bmp"}
+	config.Metadata = "../metadata"
+	config.RemoteRaw = "../remote-raw"
+	config.ProxyMode = false
+	config.Config.EnableAVIF = false
+	config.Config.Quality = 80
+	config.Config.ImageMap = map[string]string{}
+}
+
+func requestToServer(reqUrl string, app *fiber.App, ua, accept string) (*http.Response, []byte) {
+	parsedUrl, _ := url.Parse(reqUrl)
+	req := httptest.NewRequest("GET", parsedUrl.EscapedPath(), nil)
+	req.Header.Set("User-Agent", ua)
+	req.Header.Set("Accept", accept)
+	req.Header.Set("Host", parsedUrl.Host)
+	req.Host = parsedUrl.Host
+	resp, err := app.Test(req, 120000)
+	if err != nil {
+		return nil, nil
+	}
+	data, _ := io.ReadAll(resp.Body)
+	return resp, data
+}
+
+func TestServerHeaders(t *testing.T) {
+	setupParam()
+	var app = fiber.New()
+	app.Use(etag.New(etag.Config{
+		Weak: true,
+	}))
+	app.Get("/*", Convert)
+	url := "http://127.0.0.1:3333/webp_server.bmp"
+
+	// test for chrome
+	response, _ := requestToServer(url, app, chromeUA, acceptWebP)
+	defer response.Body.Close()
+	ratio := response.Header.Get("X-Compression-Rate")
+	etag := response.Header.Get("Etag")
+
+	assert.NotEqual(t, "", ratio)
+	assert.NotEqual(t, "", etag)
+
+	// test for safari
+	response, _ = requestToServer(url, app, safariUA, acceptLegacy)
+	defer response.Body.Close()
+	// ratio = response.Header.Get("X-Compression-Rate")
+	etag = response.Header.Get("Etag")
+
+	assert.NotEqual(t, "", etag)
+}
+
+func TestConvertDuplicates(t *testing.T) {
+	setupParam()
+	N := 3
+
+	var testLink = map[string]string{
+		"http://127.0.0.1:3333/webp_server.jpg":                 "image/webp",
+		"http://127.0.0.1:3333/webp_server.bmp":                 "image/webp",
+		"http://127.0.0.1:3333/webp_server.png":                 "image/webp",
+		"http://127.0.0.1:3333/empty.jpg":                       "",
+		"http://127.0.0.1:3333/png.jpg":                         "image/webp",
+		"http://127.0.0.1:3333/12314.jpg":                       "",
+		"http://127.0.0.1:3333/dir1/inside.jpg":                 "image/webp",
+		"http://127.0.0.1:3333/%e5%a4%aa%e7%a5%9e%e5%95%a6.png": "image/webp",
+		"http://127.0.0.1:3333/太神啦.png":                         "image/webp",
+	}
+
+	var app = fiber.New()
+	app.Get("/*", Convert)
+
+	// test Chrome
+	for url, respType := range testLink {
+		for i := 0; i < N; i++ {
+			resp, data := requestToServer(url, app, chromeUA, acceptWebP)
+			defer resp.Body.Close()
+			contentType := helper.GetContentType(data)
+			assert.Equal(t, respType, contentType)
+		}
+	}
+
+}
+func TestConvert(t *testing.T) {
+	setupParam()
+	// TODO: old-style test, better update it with accept headers
+	var testChromeLink = map[string]string{
+		"http://127.0.0.1:3333/webp_server.jpg":                 "image/webp",
+		"http://127.0.0.1:3333/webp_server.bmp":                 "image/webp",
+		"http://127.0.0.1:3333/webp_server.png":                 "image/webp",
+		"http://127.0.0.1:3333/empty.jpg":                       "",
+		"http://127.0.0.1:3333/png.jpg":                         "image/webp",
+		"http://127.0.0.1:3333/12314.jpg":                       "",
+		"http://127.0.0.1:3333/dir1/inside.jpg":                 "image/webp",
+		"http://127.0.0.1:3333/%e5%a4%aa%e7%a5%9e%e5%95%a6.png": "image/webp",
+		"http://127.0.0.1:3333/太神啦.png":                         "image/webp",
+	}
+
+	var testChromeAvifLink = map[string]string{
+		"http://127.0.0.1:3333/webp_server.jpg":                 "image/avif",
+		"http://127.0.0.1:3333/webp_server.bmp":                 "image/avif",
+		"http://127.0.0.1:3333/webp_server.png":                 "image/avif",
+		"http://127.0.0.1:3333/empty.jpg":                       "",
+		"http://127.0.0.1:3333/png.jpg":                         "image/avif",
+		"http://127.0.0.1:3333/12314.jpg":                       "",
+		"http://127.0.0.1:3333/dir1/inside.jpg":                 "image/avif",
+		"http://127.0.0.1:3333/%e5%a4%aa%e7%a5%9e%e5%95%a6.png": "image/avif",
+		"http://127.0.0.1:3333/太神啦.png":                         "image/avif",
+	}
+
+	var testSafariLink = map[string]string{
+		"http://127.0.0.1:3333/webp_server.jpg": "image/jpeg",
+		"http://127.0.0.1:3333/webp_server.bmp": "image/png", // png instead oft bmp because ResizeItself() uses ExportNative()
+		"http://127.0.0.1:3333/webp_server.png": "image/png",
+		"http://127.0.0.1:3333/empty.jpg":       "",
+		"http://127.0.0.1:3333/png.jpg":         "image/png",
+		"http://127.0.0.1:3333/12314.jpg":       "",
+		"http://127.0.0.1:3333/dir1/inside.jpg": "image/jpeg",
+	}
+
+	var app = fiber.New()
+	app.Get("/*", Convert)
+
+	// // test Chrome
+	for url, respType := range testChromeLink {
+		resp, data := requestToServer(url, app, chromeUA, acceptWebP)
+		defer resp.Body.Close()
+		contentType := helper.GetContentType(data)
+		assert.Equal(t, respType, contentType)
+	}
+
+	// test Safari
+	for url, respType := range testSafariLink {
+		resp, data := requestToServer(url, app, safariUA, acceptLegacy)
+		defer resp.Body.Close()
+		contentType := helper.GetContentType(data)
+		assert.Equal(t, respType, contentType)
+	}
+
+	// test Avif is processed in proxy mode
+	config.Config.EnableAVIF = true
+	for url, respType := range testChromeAvifLink {
+		resp, data := requestToServer(url, app, chromeUA, acceptAvif)
+		defer resp.Body.Close()
+		contentType := helper.GetContentType(data)
+		assert.NotNil(t, respType)
+		assert.Equal(t, respType, contentType)
+	}
+}
+
+func TestConvertNotAllowed(t *testing.T) {
+	setupParam()
+	config.Config.AllowedTypes = []string{"jpg", "png", "jpeg"}
+
+	var app = fiber.New()
+	app.Get("/*", Convert)
+
+	// not allowed, but we have the file, this should return File extension not allowed
+	url := "http://127.0.0.1:3333/webp_server.bmp"
+	resp, data := requestToServer(url, app, chromeUA, acceptWebP)
+	defer resp.Body.Close()
+	assert.Contains(t, string(data), "File extension not allowed")
+
+	// not allowed, random file
+	url = url + "hagdgd"
+	resp, data = requestToServer(url, app, chromeUA, acceptWebP)
+	defer resp.Body.Close()
+	assert.Contains(t, string(data), "File extension not allowed")
+
+}
+
+func TestConvertProxyModeBad(t *testing.T) {
+	setupParam()
+	config.ProxyMode = true
+
+	var app = fiber.New()
+	app.Get("/*", Convert)
+
+	// this is local random image, should be 404
+	url := "http://127.0.0.1:3333/webp_8888server.bmp"
+	resp, _ := requestToServer(url, app, chromeUA, acceptWebP)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusNotFound, resp.StatusCode)
+
+	// this is local random image, test using cURL, should be 404, ref: https://github.com/webp-sh/webp_server_go/issues/197
+	resp1, _ := requestToServer(url, app, curlUA, acceptWebP)
+	defer resp1.Body.Close()
+	assert.Equal(t, http.StatusNotFound, resp1.StatusCode)
+
+}
+
+func TestConvertProxyModeWork(t *testing.T) {
+	setupParam()
+	config.ProxyMode = true
+	config.Config.ImgPath = "https://webp.sh"
+
+	var app = fiber.New()
+	app.Get("/*", Convert)
+
+	url := "http://127.0.0.1:3333/images/cover.jpg"
+
+	resp, data := requestToServer(url, app, chromeUA, acceptWebP)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.Equal(t, "image/webp", helper.GetContentType(data))
+
+	// test proxyMode with Safari
+	resp, data = requestToServer(url, app, safariUA, acceptLegacy)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.Equal(t, "image/jpeg", helper.GetContentType(data))
+}
+
+func TestConvertProxyImgMap(t *testing.T) {
+	setupParam()
+	config.ProxyMode = false
+	config.Config.ImageMap = map[string]string{
+		"/2":                            "../pics/dir1",
+		"/3":                            "../pics3",        // Invalid path, does not exists
+		"www.invalid-path.com":          "https://webp.sh", // Invalid, it does not start with '/'
+		"/www.weird-path.com":           "https://webp.sh",
+		"/www.even-more-werid-path.com": "https://webp.sh/images",
+		"http://example.com":            "https://webp.sh",
+	}
+
+	var app = fiber.New()
+	app.Get("/*", Convert)
+
+	var testUrls = map[string]string{
+		"http://127.0.0.1:3333/webp_server.jpg":                        "image/webp",
+		"http://127.0.0.1:3333/2/inside.jpg":                           "image/webp",
+		"http://127.0.0.1:3333/www.weird-path.com/images/cover.jpg":    "image/webp",
+		"http://127.0.0.1:3333/www.even-more-werid-path.com/cover.jpg": "image/webp",
+		"http://example.com/images/cover.jpg":                          "image/webp",
+	}
+
+	var testUrlsLegacy = map[string]string{
+		"http://127.0.0.1:3333/webp_server.jpg": "image/jpeg",
+		"http://127.0.0.1:3333/2/inside.jpg":    "image/jpeg",
+		"http://example.com/images/cover.jpg":   "image/jpeg",
+	}
+
+	var testUrlsInvalid = map[string]string{
+		"http://127.0.0.1:3333/3/does-not-exist.jpg":         "", // Dir mapped does not exist
+		"http://127.0.0.1:3333/www.weird-path.com/cover.jpg": "", // Host mapped, final URI invalid
+	}
+
+	for url, respType := range testUrls {
+		resp, data := requestToServer(url, app, chromeUA, acceptWebP)
+		defer resp.Body.Close()
+		assert.Equal(t, http.StatusOK, resp.StatusCode)
+		assert.Equal(t, respType, helper.GetContentType(data))
+	}
+
+	// tests with Safari
+	for url, respType := range testUrlsLegacy {
+		resp, data := requestToServer(url, app, safariUA, acceptLegacy)
+		defer resp.Body.Close()
+		assert.Equal(t, http.StatusOK, resp.StatusCode)
+		assert.Equal(t, respType, helper.GetContentType(data))
+	}
+
+	for url, respType := range testUrlsInvalid {
+		resp, data := requestToServer(url, app, safariUA, acceptLegacy)
+		defer resp.Body.Close()
+		assert.Equal(t, http.StatusNotFound, resp.StatusCode)
+		assert.Equal(t, respType, helper.GetContentType(data))
+	}
+}
+
+func TestConvertProxyImgMapCWD(t *testing.T) {
+	setupParam()
+	config.ProxyMode = false
+	config.Config.ImgPath = ".." // equivalent to "" when not testing
+	config.Config.ImageMap = map[string]string{
+		"/1":                     "../pics/dir1",
+		"/2":                     "../pics",
+		"/3":                     "../pics", // Invalid path, does not exists
+		"http://www.example.com": "https://webp.sh",
+	}
+
+	var app = fiber.New()
+	app.Get("/*", Convert)
+
+	var testUrls = map[string]string{
+		"http://127.0.0.1:3333/1/inside.jpg":      "image/webp",
+		"http://127.0.0.1:3333/2/webp_server.jpg": "image/webp",
+		"http://127.0.0.1:3333/3/webp_server.jpg": "image/webp",
+		"http://www.example.com/images/cover.jpg": "image/webp",
+	}
+
+	for url, respType := range testUrls {
+		resp, data := requestToServer(url, app, chromeUA, acceptWebP)
+		defer resp.Body.Close()
+		assert.Equal(t, http.StatusOK, resp.StatusCode)
+		assert.Equal(t, respType, helper.GetContentType(data))
+	}
+}
+
+func TestConvertBigger(t *testing.T) {
+	setupParam()
+	config.Config.Quality = 100
+
+	var app = fiber.New()
+	app.Get("/*", Convert)
+
+	url := "http://127.0.0.1:3333/big.jpg"
+	resp, data := requestToServer(url, app, chromeUA, acceptWebP)
+	defer resp.Body.Close()
+	assert.Equal(t, "image/jpeg", resp.Header.Get("content-type"))
+	assert.Equal(t, "image/jpeg", helper.GetContentType(data))
+	_ = os.RemoveAll(config.Config.ExhaustPath)
+}

helper/helper.go

@@ -25,17 +25,16 @@ func svgMatcher(buf []byte) bool {
 }
 
 func GetFileContentType(filename string) string {
-	if strings.HasSuffix(filename, ".webp") {
-		return "image/webp"
-	} else if strings.HasSuffix(filename, ".avif") {
-		return "image/avif"
-	} else {
 	// raw image, need to use filetype to determine
 	buf, _ := os.ReadFile(filename)
+	return GetContentType(buf)
+}
+
+func GetContentType(buf []byte) string {
+	// raw image, need to use filetype to determine
 	kind, _ := filetype.Match(buf)
 	return kind.MIME.Value
 }
-}
 
 func FileCount(dir string) int64 {
 	var count int64 = 0
@@ -96,11 +95,11 @@ func CheckAllowedType(imgFilename string) bool {
 	return false
 }
 
-func GenOptimizedAbsPath(metadata config.MetaFile) (string, string) {
+func GenOptimizedAbsPath(metadata config.MetaFile, subdir string) (string, string) {
 	webpFilename := fmt.Sprintf("%s.webp", metadata.Id)
 	avifFilename := fmt.Sprintf("%s.avif", metadata.Id)
-	webpAbsolutePath := path.Clean(path.Join(config.Config.ExhaustPath, webpFilename))
+	webpAbsolutePath := path.Clean(path.Join(config.Config.ExhaustPath, subdir, webpFilename))
-	avifAbsolutePath := path.Clean(path.Join(config.Config.ExhaustPath, avifFilename))
+	avifAbsolutePath := path.Clean(path.Join(config.Config.ExhaustPath, subdir, avifFilename))
 	return avifAbsolutePath, webpAbsolutePath
 }
 

helper/helper_test.go

@@ -12,7 +12,6 @@ func TestMain(m *testing.M) {
 	config.LoadConfig()
 	m.Run()
 	config.ConfigPath = "config.json"
-
 }
 
 func TestFileCount(t *testing.T) {

helper/metadata.go

@@ -26,29 +26,29 @@ func getId(p string) (string, string, string) {
 	return id, path.Join(config.Config.ImgPath, parsed.Path), santizedPath
 }
 
-func ReadMetadata(p, etag string) config.MetaFile {
+func ReadMetadata(p, etag string, subdir string) config.MetaFile {
 	// try to read metadata, if we can't read, create one
 	var metadata config.MetaFile
 	var id, _, _ = getId(p)
 
-	buf, err := os.ReadFile(path.Join(config.Metadata, id+".json"))
+	buf, err := os.ReadFile(path.Join(config.Metadata, subdir, id+".json"))
 	if err != nil {
 		log.Warnf("can't read metadata: %s", err)
-		WriteMetadata(p, etag)
+		WriteMetadata(p, etag, subdir)
-		return ReadMetadata(p, etag)
+		return ReadMetadata(p, etag, subdir)
 	}
 
 	err = json.Unmarshal(buf, &metadata)
 	if err != nil {
 		log.Warnf("unmarshal metadata error, possible corrupt file, re-building...: %s", err)
-		WriteMetadata(p, etag)
+		WriteMetadata(p, etag, subdir)
-		return ReadMetadata(p, etag)
+		return ReadMetadata(p, etag, subdir)
 	}
 	return metadata
 }
 
-func WriteMetadata(p, etag string) config.MetaFile {
+func WriteMetadata(p, etag string, subdir string) config.MetaFile {
-	_ = os.Mkdir(config.Metadata, 0755)
+	_ = os.MkdirAll(path.Join(config.Metadata, subdir), 0755)
 
 	var id, filepath, sant = getId(p)
 
@@ -65,6 +65,6 @@ func WriteMetadata(p, etag string) config.MetaFile {
 	}
 
 	buf, _ := json.Marshal(data)
-	_ = os.WriteFile(path.Join(config.Metadata, data.Id+".json"), buf, 0644)
+	_ = os.WriteFile(path.Join(config.Metadata, subdir, data.Id+".json"), buf, 0644)
 	return data
 }