diff --git a/Makefile b/Makefile index e5d5b9d..24092bd 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,7 @@ test: go test -v -coverprofile=coverage.txt -covermode=atomic ./... clean: - rm -rf prefetch remote-raw exhaust tools coverage.txt + rm -rf prefetch remote-raw exhaust tools coverage.txt metadata exhaust_test docker: diff --git a/config.json b/config.json index a6f76e1..a63cc6b 100644 --- a/config.json +++ b/config.json @@ -5,6 +5,7 @@ "IMG_PATH": "./pics", "EXHAUST_PATH": "./exhaust", "ALLOWED_TYPES": ["jpg","png","jpeg","bmp","gif","svg"], + "IMG_MAP": {}, "ENABLE_AVIF": false, "ENABLE_EXTRA_PARAMS": false } diff --git a/config/config.go b/config/config.go index d225b46..f2cf158 100644 --- a/config/config.go +++ b/config/config.go @@ -6,6 +6,7 @@ import ( "os" "regexp" "runtime" + "strings" "time" "github.com/patrickmn/go-cache" @@ -17,15 +18,15 @@ const ( FiberLogFormat = "${ip} - [${time}] ${method} ${url} ${status} ${referer} ${ua}\n" WebpMax = 16383 AvifMax = 65536 - RemoteRaw = "remote-raw" - - SampleConfig = ` + HttpRegexp = `^https?://` + SampleConfig = ` { "HOST": "127.0.0.1", "PORT": "3333", "QUALITY": "80", "IMG_PATH": "./pics", "EXHAUST_PATH": "./exhaust", + "IMG_MAP": {}, "ALLOWED_TYPES": ["jpg","png","jpeg","bmp","svg"], "ENABLE_AVIF": false, "ENABLE_EXTRA_PARAMS": false @@ -60,10 +61,11 @@ var ( Config jsonFile Version = "0.9.8" WriteLock = cache.New(5*time.Minute, 10*time.Minute) + RemoteRaw = "./remote-raw" + Metadata = "./metadata" + LocalHostAlias = "local" ) -const Metadata = "metadata" - type MetaFile struct { Id string `json:"id"` // hash of below path️, also json file name id.webp Path string `json:"path"` // local: path with width and height, proxy: full url @@ -71,14 +73,15 @@ type MetaFile struct { } type jsonFile struct { - Host string `json:"HOST"` - Port string `json:"PORT"` - ImgPath string `json:"IMG_PATH"` - Quality int `json:"QUALITY,string"` - AllowedTypes []string `json:"ALLOWED_TYPES"` - ExhaustPath string `json:"EXHAUST_PATH"` - EnableAVIF bool `json:"ENABLE_AVIF"` - EnableExtraParams bool `json:"ENABLE_EXTRA_PARAMS"` + Host string `json:"HOST"` + Port string `json:"PORT"` + ImgPath string `json:"IMG_PATH"` + Quality int `json:"QUALITY,string"` + AllowedTypes []string `json:"ALLOWED_TYPES"` + ImageMap map[string]string `json:"IMG_MAP"` + ExhaustPath string `json:"EXHAUST_PATH"` + EnableAVIF bool `json:"ENABLE_AVIF"` + EnableExtraParams bool `json:"ENABLE_EXTRA_PARAMS"` } func init() { @@ -99,6 +102,22 @@ func LoadConfig() { _ = decoder.Decode(&Config) _ = jsonObject.Close() switchProxyMode() + Config.ImageMap = parseImgMap(Config.ImageMap) +} + +func parseImgMap(imgMap map[string]string) map[string]string { + var parsedImgMap = map[string]string{} + httpRegexpMatcher := regexp.MustCompile(HttpRegexp) + for uriMap, uriMapTarget := range imgMap { + if httpRegexpMatcher.Match([]byte(uriMap)) || strings.HasPrefix(uriMap, "/") { + // Valid + parsedImgMap[uriMap] = uriMapTarget + } else { + // Invalid + log.Warnf("IMG_MAP key '%s' does matches '%s' or starts with '/' - skipped", uriMap, HttpRegexp) + } + } + return parsedImgMap } type ExtraParams struct { @@ -107,8 +126,10 @@ type ExtraParams struct { } func switchProxyMode() { - matched, _ := regexp.MatchString(`^https?://`, Config.ImgPath) + matched, _ := regexp.MatchString(HttpRegexp, Config.ImgPath) if matched { + // Enable proxy based on ImgPath should be deprecated in future versions + log.Warn("Enable proxy based on ImgPath will be deprecated in future versions. Use IMG_MAP config options instead") ProxyMode = true } } diff --git a/config/config_test.go b/config/config_test.go index dbb82eb..12b9e22 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -19,6 +19,7 @@ func TestLoadConfig(t *testing.T) { assert.Equal(t, Config.Port, "3333") assert.Equal(t, Config.Quality, 80) assert.Equal(t, Config.ImgPath, "./pics") + assert.Equal(t, Config.ImageMap, map[string]string{}) assert.Equal(t, Config.ExhaustPath, "./exhaust") } @@ -29,3 +30,26 @@ func TestSwitchProxyMode(t *testing.T) { switchProxyMode() assert.True(t, ProxyMode) } + +func TestParseImgMap(t *testing.T) { + empty := map[string]string{} + good := map[string]string{ + "/1": "../pics/dir1", + "http://example.com": "../pics", + "https://example.com": "../pics", + } + bad := map[string]string{ + "1": "../pics/dir1", + "httpx://example.com": "../pics", + "ftp://example.com": "../pics", + } + + assert.Equal(t, empty, parseImgMap(empty)) + assert.Equal(t, empty, parseImgMap(bad)) + assert.Equal(t, good, parseImgMap(good)) + + for k, v := range good { + bad[k] = v + } + assert.Equal(t, good, parseImgMap(bad)) +} \ No newline at end of file diff --git a/encoder/encoder.go b/encoder/encoder.go index f7b8586..116b186 100644 --- a/encoder/encoder.go +++ b/encoder/encoder.go @@ -86,9 +86,20 @@ func ConvertFilter(raw, avifPath, webpPath string, extraParams config.ExtraParam func ResizeItself(raw, dest string, extraParams config.ExtraParams) { log.Infof("Resize %s itself to %s", raw, dest) - img, _ := vips.LoadImageFromFile(raw, &vips.ImportParams{ + + // we need to create dir first + var err = os.MkdirAll(path.Dir(dest), 0755) + if err != nil { + log.Error(err.Error()) + } + + img, err := vips.LoadImageFromFile(raw, &vips.ImportParams{ FailOnError: boolFalse, }) + if err != nil { + log.Warnf("Could not load %s: %s", raw, err) + return + } _ = resizeImage(img, extraParams) buf, _, _ := img.ExportNative() _ = os.WriteFile(dest, buf, 0600) diff --git a/encoder/prefetch.go b/encoder/prefetch.go index dfb4256..c841aea 100644 --- a/encoder/prefetch.go +++ b/encoder/prefetch.go @@ -34,8 +34,8 @@ func PrefetchImages() { return nil } // RawImagePath string, ImgFilename string, reqURI string - metadata := helper.ReadMetadata(picAbsPath, "") - avif, webp := helper.GenOptimizedAbsPath(metadata) + metadata := helper.ReadMetadata(picAbsPath, "", config.LocalHostAlias) + avif, webp := helper.GenOptimizedAbsPath(metadata, config.LocalHostAlias) _ = os.MkdirAll(path.Dir(avif), 0755) log.Infof("Prefetching %s", picAbsPath) go ConvertFilter(picAbsPath, avif, webp, config.ExtraParams{Width: 0, Height: 0}, finishChan) diff --git a/go.mod b/go.mod index 036e519..2d46c5c 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,8 @@ require ( github.com/cespare/xxhash v1.1.0 github.com/davidbyttow/govips/v2 v2.13.0 github.com/gofiber/fiber/v2 v2.48.0 - github.com/h2non/filetype v1.1.3 + github.com/h2non/filetype v1.1.4-0.20230123234534-cfcd7d097bc4 + github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c github.com/patrickmn/go-cache v2.1.0+incompatible github.com/schollz/progressbar/v3 v3.13.1 github.com/sirupsen/logrus v1.9.3 @@ -18,7 +19,6 @@ require ( github.com/andybalholm/brotli v1.0.5 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/google/uuid v1.3.0 // indirect - github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c // indirect github.com/klauspost/compress v1.16.3 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect diff --git a/go.sum b/go.sum index a8fb305..1520937 100644 --- a/go.sum +++ b/go.sum @@ -13,8 +13,8 @@ github.com/gofiber/fiber/v2 v2.48.0 h1:cRVMCb9aUJDsyHxGFLwz/sGzDggdailZZyptU9F9c github.com/gofiber/fiber/v2 v2.48.0/go.mod h1:xqJgfqrc23FJuqGOW6DVgi3HyZEm2Mn9pRqUb2kHSX8= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= -github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= +github.com/h2non/filetype v1.1.4-0.20230123234534-cfcd7d097bc4 h1:k7FGP5I7raiaC3aAzCLddcoxzboIrOm6/FVRXjp/5JM= +github.com/h2non/filetype v1.1.4-0.20230123234534-cfcd7d097bc4/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c h1:fEE5/5VNnYUoBOj2I9TP8Jc+a7lge3QWn9DKE7NCwfc= github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c/go.mod h1:ObS/W+h8RYb1Y7fYivughjxojTmIu5iAIjSrSLCLeqE= github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= diff --git a/handler/remote.go b/handler/remote.go index 66f9665..dc1c699 100644 --- a/handler/remote.go +++ b/handler/remote.go @@ -75,18 +75,18 @@ func downloadFile(filepath string, url string) { } -func fetchRemoteImg(url string) config.MetaFile { +func fetchRemoteImg(url string, subdir string) config.MetaFile { // url is https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200 // How do we know if the remote img is changed? we're using hash(etag+length) log.Infof("Remote Addr is %s, pinging for info...", url) etag := pingURL(url) - metadata := helper.ReadMetadata(url, etag) - localRawImagePath := path.Join(config.RemoteRaw, metadata.Id) + metadata := helper.ReadMetadata(url, etag, subdir) + localRawImagePath := path.Join(config.RemoteRaw, subdir, metadata.Id) if !helper.ImageExists(localRawImagePath) || metadata.Checksum != helper.HashString(etag) { // remote file has changed or local file not exists log.Info("Remote file not found in remote-raw, re-fetching...") - cleanProxyCache(path.Join(config.Config.ExhaustPath, metadata.Id+"*")) + cleanProxyCache(path.Join(config.Config.ExhaustPath, subdir, metadata.Id+"*")) downloadFile(localRawImagePath, url) } return metadata diff --git a/handler/router.go b/handler/router.go index 71d1fd9..3ef1f2c 100644 --- a/handler/router.go +++ b/handler/router.go @@ -3,6 +3,8 @@ package handler import ( "net/http" "net/url" + "regexp" + "strings" "webp_server_go/config" "webp_server_go/encoder" "webp_server_go/helper" @@ -21,11 +23,20 @@ func Convert(c *fiber.Ctx) error { // 3. pass it to encoder, get the result, send it back var ( + reqHostname = c.Hostname() + reqHost = c.Protocol() + "://" + reqHostname // http://www.example.com:8000 reqURI, _ = url.QueryUnescape(c.Path()) // /mypic/123.jpg reqURIwithQuery, _ = url.QueryUnescape(c.OriginalURL()) // /mypic/123.jpg?someother=200&somebugs=200 filename = path.Base(reqURI) + realRemoteAddr = "" + targetHostName = config.LocalHostAlias + targetHost = config.Config.ImgPath + proxyMode = config.ProxyMode + mapMode = false ) + log.Debugf("Incoming connection from %s %s %s", c.IP(), reqHostname, reqURIwithQuery) + if !helper.CheckAllowedType(filename) { msg := "File extension not allowed! " + filename log.Warn(msg) @@ -47,29 +58,84 @@ func Convert(c *fiber.Ctx) error { Height: height, } + // Rewrite the target backend if a mapping rule matches the hostname + if hostMap, hostMapFound := config.Config.ImageMap[reqHost]; hostMapFound { + log.Debugf("Found host mapping %s -> %s", reqHostname, hostMap) + targetHostUrl, _ := url.Parse(hostMap) + targetHostName = targetHostUrl.Host + targetHost = targetHostUrl.Scheme + "://" + targetHostUrl.Host + proxyMode = true + } else { + // There's not matching host mapping, now check for any URI map that apply + httpRegexpMatcher := regexp.MustCompile(config.HttpRegexp) + for uriMap, uriMapTarget := range config.Config.ImageMap { + if strings.HasPrefix(reqURI, uriMap) { + log.Debugf("Found URI mapping %s -> %s", uriMap, uriMapTarget) + mapMode = true + + // if uriMapTarget we use the proxy mode to fetch the remote + if httpRegexpMatcher.Match([]byte(uriMapTarget)) { + targetHostUrl, _ := url.Parse(uriMapTarget) + targetHostName = targetHostUrl.Host + targetHost = targetHostUrl.Scheme + "://" + targetHostUrl.Host + reqURI = strings.Replace(reqURI, uriMap, targetHostUrl.Path, 1) + reqURIwithQuery = strings.Replace(reqURIwithQuery, uriMap, targetHostUrl.Path, 1) + proxyMode = true + } else { + reqURI = strings.Replace(reqURI, uriMap, uriMapTarget, 1) + reqURIwithQuery = strings.Replace(reqURIwithQuery, uriMap, uriMapTarget, 1) + } + break + } + } + + } + + if proxyMode { + + if !mapMode { + // Don't deal with the encoding to avoid upstream compatibilities + reqURI = c.Path() + reqURIwithQuery = c.OriginalURL() + } + + log.Tracef("reqURIwithQuery is %s", reqURIwithQuery) + + // Replace host in the URL + // realRemoteAddr = strings.Replace(reqURIwithQuery, reqHost, targetHost, 1) + realRemoteAddr = targetHost + reqURIwithQuery + log.Debugf("realRemoteAddr is %s", realRemoteAddr) + } + var rawImageAbs string var metadata = config.MetaFile{} - if config.ProxyMode { + if proxyMode { // this is proxyMode, we'll have to use this url to download and save it to local path, which also gives us rawImageAbs // https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200 - metadata = fetchRemoteImg(config.Config.ImgPath + reqURIwithQuery) - rawImageAbs = path.Join(config.RemoteRaw, metadata.Id) + + metadata = fetchRemoteImg(realRemoteAddr, targetHostName) + rawImageAbs = path.Join(config.RemoteRaw, targetHostName, metadata.Id) } else { // not proxyMode, we'll use local path - metadata = helper.ReadMetadata(reqURIwithQuery, "") - rawImageAbs = path.Join(config.Config.ImgPath, reqURI) + metadata = helper.ReadMetadata(reqURIwithQuery, "", targetHostName) + if !mapMode { + // by default images are hosted in ImgPath + rawImageAbs = path.Join(config.Config.ImgPath, reqURI) + } else { + rawImageAbs = reqURI + } // detect if source file has changed if metadata.Checksum != helper.HashFile(rawImageAbs) { log.Info("Source file has changed, re-encoding...") - helper.WriteMetadata(reqURIwithQuery, "") - cleanProxyCache(path.Join(config.Config.ExhaustPath, metadata.Id)) + helper.WriteMetadata(reqURIwithQuery, "", targetHostName) + cleanProxyCache(path.Join(config.Config.ExhaustPath, targetHostName, metadata.Id)) } } goodFormat := helper.GuessSupportedFormat(&c.Request().Header) // resize itself and return if only one format(raw) is supported if len(goodFormat) == 1 { - dest := path.Join(config.Config.ExhaustPath, metadata.Id) + dest := path.Join(config.Config.ExhaustPath, targetHostName, metadata.Id) if !helper.ImageExists(dest) { encoder.ResizeItself(rawImageAbs, dest, extraParams) } @@ -85,7 +151,7 @@ func Convert(c *fiber.Ctx) error { return nil } - avifAbs, webpAbs := helper.GenOptimizedAbsPath(metadata) + avifAbs, webpAbs := helper.GenOptimizedAbsPath(metadata, targetHostName) encoder.ConvertFilter(rawImageAbs, avifAbs, webpAbs, extraParams, nil) var availableFiles = []string{rawImageAbs} diff --git a/handler/router_test.go b/handler/router_test.go new file mode 100644 index 0000000..aa397de --- /dev/null +++ b/handler/router_test.go @@ -0,0 +1,341 @@ +package handler + +import ( + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "testing" + "webp_server_go/config" + "webp_server_go/helper" + + "github.com/gofiber/fiber/v2" + "github.com/gofiber/fiber/v2/middleware/etag" + "github.com/stretchr/testify/assert" +) + +var ( + chromeUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36" + acceptWebP = "image/webp,image/apng,image/*,*/*;q=0.8" + acceptAvif = "image/avif,image/*,*/*;q=0.8" + acceptLegacy = "image/jpeg" + safariUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1 Safari/605.1.15" + curlUA = "curl/7.64.1" +) + +func setupParam() { + // setup parameters here... + config.Config.ImgPath = "../pics" + config.Config.ExhaustPath = "../exhaust_test" + config.Config.AllowedTypes = []string{"jpg", "png", "jpeg", "bmp"} + config.Metadata = "../metadata" + config.RemoteRaw = "../remote-raw" + config.ProxyMode = false + config.Config.EnableAVIF = false + config.Config.Quality = 80 + config.Config.ImageMap = map[string]string{} +} + +func requestToServer(reqUrl string, app *fiber.App, ua, accept string) (*http.Response, []byte) { + parsedUrl, _ := url.Parse(reqUrl) + req := httptest.NewRequest("GET", parsedUrl.EscapedPath(), nil) + req.Header.Set("User-Agent", ua) + req.Header.Set("Accept", accept) + req.Header.Set("Host", parsedUrl.Host) + req.Host = parsedUrl.Host + resp, err := app.Test(req, 120000) + if err != nil { + return nil, nil + } + data, _ := io.ReadAll(resp.Body) + return resp, data +} + +func TestServerHeaders(t *testing.T) { + setupParam() + var app = fiber.New() + app.Use(etag.New(etag.Config{ + Weak: true, + })) + app.Get("/*", Convert) + url := "http://127.0.0.1:3333/webp_server.bmp" + + // test for chrome + response, _ := requestToServer(url, app, chromeUA, acceptWebP) + defer response.Body.Close() + ratio := response.Header.Get("X-Compression-Rate") + etag := response.Header.Get("Etag") + + assert.NotEqual(t, "", ratio) + assert.NotEqual(t, "", etag) + + // test for safari + response, _ = requestToServer(url, app, safariUA, acceptLegacy) + defer response.Body.Close() + // ratio = response.Header.Get("X-Compression-Rate") + etag = response.Header.Get("Etag") + + assert.NotEqual(t, "", etag) +} + +func TestConvertDuplicates(t *testing.T) { + setupParam() + N := 3 + + var testLink = map[string]string{ + "http://127.0.0.1:3333/webp_server.jpg": "image/webp", + "http://127.0.0.1:3333/webp_server.bmp": "image/webp", + "http://127.0.0.1:3333/webp_server.png": "image/webp", + "http://127.0.0.1:3333/empty.jpg": "", + "http://127.0.0.1:3333/png.jpg": "image/webp", + "http://127.0.0.1:3333/12314.jpg": "", + "http://127.0.0.1:3333/dir1/inside.jpg": "image/webp", + "http://127.0.0.1:3333/%e5%a4%aa%e7%a5%9e%e5%95%a6.png": "image/webp", + "http://127.0.0.1:3333/太神啦.png": "image/webp", + } + + var app = fiber.New() + app.Get("/*", Convert) + + // test Chrome + for url, respType := range testLink { + for i := 0; i < N; i++ { + resp, data := requestToServer(url, app, chromeUA, acceptWebP) + defer resp.Body.Close() + contentType := helper.GetContentType(data) + assert.Equal(t, respType, contentType) + } + } + +} +func TestConvert(t *testing.T) { + setupParam() + // TODO: old-style test, better update it with accept headers + var testChromeLink = map[string]string{ + "http://127.0.0.1:3333/webp_server.jpg": "image/webp", + "http://127.0.0.1:3333/webp_server.bmp": "image/webp", + "http://127.0.0.1:3333/webp_server.png": "image/webp", + "http://127.0.0.1:3333/empty.jpg": "", + "http://127.0.0.1:3333/png.jpg": "image/webp", + "http://127.0.0.1:3333/12314.jpg": "", + "http://127.0.0.1:3333/dir1/inside.jpg": "image/webp", + "http://127.0.0.1:3333/%e5%a4%aa%e7%a5%9e%e5%95%a6.png": "image/webp", + "http://127.0.0.1:3333/太神啦.png": "image/webp", + } + + var testChromeAvifLink = map[string]string{ + "http://127.0.0.1:3333/webp_server.jpg": "image/avif", + "http://127.0.0.1:3333/webp_server.bmp": "image/avif", + "http://127.0.0.1:3333/webp_server.png": "image/avif", + "http://127.0.0.1:3333/empty.jpg": "", + "http://127.0.0.1:3333/png.jpg": "image/avif", + "http://127.0.0.1:3333/12314.jpg": "", + "http://127.0.0.1:3333/dir1/inside.jpg": "image/avif", + "http://127.0.0.1:3333/%e5%a4%aa%e7%a5%9e%e5%95%a6.png": "image/avif", + "http://127.0.0.1:3333/太神啦.png": "image/avif", + } + + var testSafariLink = map[string]string{ + "http://127.0.0.1:3333/webp_server.jpg": "image/jpeg", + "http://127.0.0.1:3333/webp_server.bmp": "image/png", // png instead oft bmp because ResizeItself() uses ExportNative() + "http://127.0.0.1:3333/webp_server.png": "image/png", + "http://127.0.0.1:3333/empty.jpg": "", + "http://127.0.0.1:3333/png.jpg": "image/png", + "http://127.0.0.1:3333/12314.jpg": "", + "http://127.0.0.1:3333/dir1/inside.jpg": "image/jpeg", + } + + var app = fiber.New() + app.Get("/*", Convert) + + // // test Chrome + for url, respType := range testChromeLink { + resp, data := requestToServer(url, app, chromeUA, acceptWebP) + defer resp.Body.Close() + contentType := helper.GetContentType(data) + assert.Equal(t, respType, contentType) + } + + // test Safari + for url, respType := range testSafariLink { + resp, data := requestToServer(url, app, safariUA, acceptLegacy) + defer resp.Body.Close() + contentType := helper.GetContentType(data) + assert.Equal(t, respType, contentType) + } + + // test Avif is processed in proxy mode + config.Config.EnableAVIF = true + for url, respType := range testChromeAvifLink { + resp, data := requestToServer(url, app, chromeUA, acceptAvif) + defer resp.Body.Close() + contentType := helper.GetContentType(data) + assert.NotNil(t, respType) + assert.Equal(t, respType, contentType) + } +} + +func TestConvertNotAllowed(t *testing.T) { + setupParam() + config.Config.AllowedTypes = []string{"jpg", "png", "jpeg"} + + var app = fiber.New() + app.Get("/*", Convert) + + // not allowed, but we have the file, this should return File extension not allowed + url := "http://127.0.0.1:3333/webp_server.bmp" + resp, data := requestToServer(url, app, chromeUA, acceptWebP) + defer resp.Body.Close() + assert.Contains(t, string(data), "File extension not allowed") + + // not allowed, random file + url = url + "hagdgd" + resp, data = requestToServer(url, app, chromeUA, acceptWebP) + defer resp.Body.Close() + assert.Contains(t, string(data), "File extension not allowed") + +} + +func TestConvertProxyModeBad(t *testing.T) { + setupParam() + config.ProxyMode = true + + var app = fiber.New() + app.Get("/*", Convert) + + // this is local random image, should be 404 + url := "http://127.0.0.1:3333/webp_8888server.bmp" + resp, _ := requestToServer(url, app, chromeUA, acceptWebP) + defer resp.Body.Close() + assert.Equal(t, http.StatusNotFound, resp.StatusCode) + + // this is local random image, test using cURL, should be 404, ref: https://github.com/webp-sh/webp_server_go/issues/197 + resp1, _ := requestToServer(url, app, curlUA, acceptWebP) + defer resp1.Body.Close() + assert.Equal(t, http.StatusNotFound, resp1.StatusCode) + +} + +func TestConvertProxyModeWork(t *testing.T) { + setupParam() + config.ProxyMode = true + config.Config.ImgPath = "https://webp.sh" + + var app = fiber.New() + app.Get("/*", Convert) + + url := "http://127.0.0.1:3333/images/cover.jpg" + + resp, data := requestToServer(url, app, chromeUA, acceptWebP) + defer resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, "image/webp", helper.GetContentType(data)) + + // test proxyMode with Safari + resp, data = requestToServer(url, app, safariUA, acceptLegacy) + defer resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, "image/jpeg", helper.GetContentType(data)) +} + +func TestConvertProxyImgMap(t *testing.T) { + setupParam() + config.ProxyMode = false + config.Config.ImageMap = map[string]string{ + "/2": "../pics/dir1", + "/3": "../pics3", // Invalid path, does not exists + "www.invalid-path.com": "https://webp.sh", // Invalid, it does not start with '/' + "/www.weird-path.com": "https://webp.sh", + "/www.even-more-werid-path.com": "https://webp.sh/images", + "http://example.com": "https://webp.sh", + } + + var app = fiber.New() + app.Get("/*", Convert) + + var testUrls = map[string]string{ + "http://127.0.0.1:3333/webp_server.jpg": "image/webp", + "http://127.0.0.1:3333/2/inside.jpg": "image/webp", + "http://127.0.0.1:3333/www.weird-path.com/images/cover.jpg": "image/webp", + "http://127.0.0.1:3333/www.even-more-werid-path.com/cover.jpg": "image/webp", + "http://example.com/images/cover.jpg": "image/webp", + } + + var testUrlsLegacy = map[string]string{ + "http://127.0.0.1:3333/webp_server.jpg": "image/jpeg", + "http://127.0.0.1:3333/2/inside.jpg": "image/jpeg", + "http://example.com/images/cover.jpg": "image/jpeg", + } + + var testUrlsInvalid = map[string]string{ + "http://127.0.0.1:3333/3/does-not-exist.jpg": "", // Dir mapped does not exist + "http://127.0.0.1:3333/www.weird-path.com/cover.jpg": "", // Host mapped, final URI invalid + } + + for url, respType := range testUrls { + resp, data := requestToServer(url, app, chromeUA, acceptWebP) + defer resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, respType, helper.GetContentType(data)) + } + + // tests with Safari + for url, respType := range testUrlsLegacy { + resp, data := requestToServer(url, app, safariUA, acceptLegacy) + defer resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, respType, helper.GetContentType(data)) + } + + for url, respType := range testUrlsInvalid { + resp, data := requestToServer(url, app, safariUA, acceptLegacy) + defer resp.Body.Close() + assert.Equal(t, http.StatusNotFound, resp.StatusCode) + assert.Equal(t, respType, helper.GetContentType(data)) + } +} + +func TestConvertProxyImgMapCWD(t *testing.T) { + setupParam() + config.ProxyMode = false + config.Config.ImgPath = ".." // equivalent to "" when not testing + config.Config.ImageMap = map[string]string{ + "/1": "../pics/dir1", + "/2": "../pics", + "/3": "../pics", // Invalid path, does not exists + "http://www.example.com": "https://webp.sh", + } + + var app = fiber.New() + app.Get("/*", Convert) + + var testUrls = map[string]string{ + "http://127.0.0.1:3333/1/inside.jpg": "image/webp", + "http://127.0.0.1:3333/2/webp_server.jpg": "image/webp", + "http://127.0.0.1:3333/3/webp_server.jpg": "image/webp", + "http://www.example.com/images/cover.jpg": "image/webp", + } + + for url, respType := range testUrls { + resp, data := requestToServer(url, app, chromeUA, acceptWebP) + defer resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, respType, helper.GetContentType(data)) + } +} + +func TestConvertBigger(t *testing.T) { + setupParam() + config.Config.Quality = 100 + + var app = fiber.New() + app.Get("/*", Convert) + + url := "http://127.0.0.1:3333/big.jpg" + resp, data := requestToServer(url, app, chromeUA, acceptWebP) + defer resp.Body.Close() + assert.Equal(t, "image/jpeg", resp.Header.Get("content-type")) + assert.Equal(t, "image/jpeg", helper.GetContentType(data)) + _ = os.RemoveAll(config.Config.ExhaustPath) +} diff --git a/helper/helper.go b/helper/helper.go index 5d81c1d..89aa870 100644 --- a/helper/helper.go +++ b/helper/helper.go @@ -25,16 +25,15 @@ func svgMatcher(buf []byte) bool { } func GetFileContentType(filename string) string { - if strings.HasSuffix(filename, ".webp") { - return "image/webp" - } else if strings.HasSuffix(filename, ".avif") { - return "image/avif" - } else { - // raw image, need to use filetype to determine - buf, _ := os.ReadFile(filename) - kind, _ := filetype.Match(buf) - return kind.MIME.Value - } + // raw image, need to use filetype to determine + buf, _ := os.ReadFile(filename) + return GetContentType(buf) +} + +func GetContentType(buf []byte) string { + // raw image, need to use filetype to determine + kind, _ := filetype.Match(buf) + return kind.MIME.Value } func FileCount(dir string) int64 { @@ -96,11 +95,11 @@ func CheckAllowedType(imgFilename string) bool { return false } -func GenOptimizedAbsPath(metadata config.MetaFile) (string, string) { +func GenOptimizedAbsPath(metadata config.MetaFile, subdir string) (string, string) { webpFilename := fmt.Sprintf("%s.webp", metadata.Id) avifFilename := fmt.Sprintf("%s.avif", metadata.Id) - webpAbsolutePath := path.Clean(path.Join(config.Config.ExhaustPath, webpFilename)) - avifAbsolutePath := path.Clean(path.Join(config.Config.ExhaustPath, avifFilename)) + webpAbsolutePath := path.Clean(path.Join(config.Config.ExhaustPath, subdir, webpFilename)) + avifAbsolutePath := path.Clean(path.Join(config.Config.ExhaustPath, subdir, avifFilename)) return avifAbsolutePath, webpAbsolutePath } diff --git a/helper/helper_test.go b/helper/helper_test.go index 1469918..efe80a9 100644 --- a/helper/helper_test.go +++ b/helper/helper_test.go @@ -12,7 +12,6 @@ func TestMain(m *testing.M) { config.LoadConfig() m.Run() config.ConfigPath = "config.json" - } func TestFileCount(t *testing.T) { diff --git a/helper/metadata.go b/helper/metadata.go index 75f7599..2771146 100644 --- a/helper/metadata.go +++ b/helper/metadata.go @@ -26,29 +26,29 @@ func getId(p string) (string, string, string) { return id, path.Join(config.Config.ImgPath, parsed.Path), santizedPath } -func ReadMetadata(p, etag string) config.MetaFile { +func ReadMetadata(p, etag string, subdir string) config.MetaFile { // try to read metadata, if we can't read, create one var metadata config.MetaFile var id, _, _ = getId(p) - buf, err := os.ReadFile(path.Join(config.Metadata, id+".json")) + buf, err := os.ReadFile(path.Join(config.Metadata, subdir, id+".json")) if err != nil { log.Warnf("can't read metadata: %s", err) - WriteMetadata(p, etag) - return ReadMetadata(p, etag) + WriteMetadata(p, etag, subdir) + return ReadMetadata(p, etag, subdir) } err = json.Unmarshal(buf, &metadata) if err != nil { log.Warnf("unmarshal metadata error, possible corrupt file, re-building...: %s", err) - WriteMetadata(p, etag) - return ReadMetadata(p, etag) + WriteMetadata(p, etag, subdir) + return ReadMetadata(p, etag, subdir) } return metadata } -func WriteMetadata(p, etag string) config.MetaFile { - _ = os.Mkdir(config.Metadata, 0755) +func WriteMetadata(p, etag string, subdir string) config.MetaFile { + _ = os.MkdirAll(path.Join(config.Metadata, subdir), 0755) var id, filepath, sant = getId(p) @@ -65,6 +65,6 @@ func WriteMetadata(p, etag string) config.MetaFile { } buf, _ := json.Marshal(data) - _ = os.WriteFile(path.Join(config.Metadata, data.Id+".json"), buf, 0644) + _ = os.WriteFile(path.Join(config.Metadata, subdir, data.Id+".json"), buf, 0644) return data }