From 2d2df5571dd0d462eb8c1a3109fb75b8731147ca Mon Sep 17 00:00:00 2001 From: Nova Kwok Date: Thu, 6 Aug 2020 14:55:20 +0800 Subject: [PATCH] Feature: WebP Proxy (#51) * feat-webp-proxy * Fix panic on clear Cache, modified output. * Optimize etag fetch logic * Update README for new docs website. * Bump version to 0.2.0. --- README.md | 7 ++- helper.go | 53 ++++++++++++++++++- router.go | 137 +++++++++++++++++++++++++++++++++---------------- webp-server.go | 25 ++++++--- 4 files changed, 166 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index 096a806..4dc45e7 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@

-[Documentation](https://webp.sh/docs/) | [Website](https://webp.sh/) +[Documentation](https://docs.webp.sh/) | [Website](https://webp.sh/) This is a Server based on Golang, which allows you to serve WebP images on the fly. It will convert `jpg,jpeg,png` files by default, this can be customized by editing the `config.json`.. @@ -18,7 +18,7 @@ It will convert `jpg,jpeg,png` files by default, this can be customized by editi ## Simple Usage Steps ### 1. Download or build the binary -Download the `webp-server` from [release](https://github.com/n0vad3v/webp_server_go/releases) page. +Download the `webp-server` from [release](https://github.com/webp-sh/webp_server_go/releases) page. ### 2. Dump config file @@ -67,8 +67,7 @@ Let Nginx to `proxy_pass http://localhost:3333/;`, and your webp-server is on-th ## Advanced Usage -For supervisor, Docker sections, please read our documentation at [https://webp.sh/docs/](https://webp.sh/docs/) - +For supervisor, Docker sections, please read our documentation at [https://docs.webp.sh/](https://docs.webp.sh/) ## License diff --git a/helper.go b/helper.go index 3b8ccae..8c806e3 100644 --- a/helper.go +++ b/helper.go @@ -3,6 +3,7 @@ package main import ( "fmt" "hash/crc32" + "io" "io/ioutil" "net/http" "os" @@ -48,6 +49,56 @@ func ImageExists(filename string) bool { return !info.IsDir() } +// Check for remote filepath, e.g: https://test.webp.sh/node.png +// return StatusCode, etagValue +func GetRemoteImageInfo(fileUrl string) (int, string) { + res, err := http.Head(fileUrl) + if err != nil { + log.Fatal("Connection to remote error!") + } + if res.StatusCode != 404 { + etagValue := res.Header.Get("etag") + if etagValue == "" { + log.Info("Remote didn't return etag in header, please check.") + } else { + return 200, etagValue + } + } + return res.StatusCode, "" +} + +func FetchRemoteImage(filepath string, url string) error { + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + _ = os.MkdirAll(path.Dir(filepath), 0755) + out, err := os.Create(filepath) + if err != nil { + return err + } + defer out.Close() + + _, err = io.Copy(out, resp.Body) + return err +} + +// Given /path/to/node.png +// Delete /path/to/node.png* +func CleanProxyCache(cacheImagePath string) { + // Delete /node.png* + files, err := filepath.Glob(cacheImagePath + "*") + if err != nil { + fmt.Println(err) + } + for _, f := range files { + if err := os.Remove(f); err != nil { + log.Info(err) + } + } +} + func GenWebpAbs(RawImagePath string, ExhaustPath string, ImgFilename string, reqURI string) (string, string) { // get file mod time STAT, err := os.Stat(RawImagePath) @@ -56,7 +107,7 @@ func GenWebpAbs(RawImagePath string, ExhaustPath string, ImgFilename string, req } ModifiedTime := STAT.ModTime().Unix() // webpFilename: abc.jpg.png -> abc.jpg.png1582558990.webp - var WebpFilename = fmt.Sprintf("%s.%d.webp", ImgFilename, ModifiedTime) + WebpFilename := fmt.Sprintf("%s.%d.webp", ImgFilename, ModifiedTime) cwd, _ := os.Getwd() // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp diff --git a/router.go b/router.go index f0183ee..1400f0d 100644 --- a/router.go +++ b/router.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "os" "path" "path/filepath" @@ -12,10 +13,10 @@ import ( "github.com/gofiber/fiber" ) -func Convert(ImgPath string, ExhaustPath string, AllowedTypes []string, QUALITY string) func(c *fiber.Ctx) { +func Convert(ImgPath string, ExhaustPath string, AllowedTypes []string, QUALITY string, proxyMode bool) func(c *fiber.Ctx) { return func(c *fiber.Ctx) { //basic vars - var reqURI = c.Path() // mypic/123.jpg + var reqURI = c.Path() // /mypic/123.jpg var RawImageAbs = path.Join(ImgPath, reqURI) // /home/xxx/mypic/123.jpg var ImgFilename = path.Base(reqURI) // pure filename, 123.jpg var finalFile string // We'll only need one c.sendFile() @@ -57,52 +58,102 @@ func Convert(ImgPath string, ExhaustPath string, AllowedTypes []string, QUALITY return } - // Check the original image for existence, - if !ImageExists(RawImageAbs) { - msg := "Image not found!" - c.Send(msg) - log.Warn(msg) - c.SendStatus(404) - return - } + // Start Proxy Mode + if proxyMode { + // https://test.webp.sh/node.png + realRemoteAddr := ImgPath + reqURI + // Ping Remote for status code and etag info - _, WebpAbsPath := GenWebpAbs(RawImageAbs, ExhaustPath, ImgFilename, reqURI) - - if ImageExists(WebpAbsPath) { - finalFile = WebpAbsPath - } else { - // we don't have abc.jpg.png1582558990.webp - // delete the old pic and convert a new one. - // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp - destHalfFile := path.Clean(path.Join(WebpAbsPath, path.Dir(reqURI), ImgFilename)) - matches, err := filepath.Glob(destHalfFile + "*") - if err != nil { - log.Error(err.Error()) - } else { - // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558100.webp <- older ones will be removed - // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp <- keep the latest one - for _, p := range matches { - if strings.Compare(destHalfFile, p) != 0 { - _ = os.Remove(p) + // If status code is 200 + // Check for local /node.png-etag- + // if exist + // Send local cache + // else + // Delete local /node.png* + // Fetch and convert to /node.png-etag- + // Send local cache + // else status code is 404 + // Delete /node.png* + // Send 404 + fmt.Println("Remote Addr is " + realRemoteAddr + ", fetching..") + statusCode, etagValue := GetRemoteImageInfo(realRemoteAddr) + if statusCode == 200 { + // Check local path: /node.png-etag- + localEtagImagePath := ExhaustPath + reqURI + "-etag-" + etagValue + if ImageExists(localEtagImagePath) { + c.SendFile(localEtagImagePath) + } else { + // Temporary store of remote file. + // ./remote-raw/node.png + CleanProxyCache(ExhaustPath + reqURI + "*") + localRemoteTmpPath := "./remote-raw" + reqURI + FetchRemoteImage(localRemoteTmpPath, realRemoteAddr) + q, _ := strconv.ParseFloat(QUALITY, 32) + _ = os.MkdirAll(path.Dir(localEtagImagePath), 0755) + err := WebpEncoder(localRemoteTmpPath, localEtagImagePath, float32(q), true, nil) + if err != nil { + fmt.Println(err) } + c.SendFile(localEtagImagePath) } - } - - //for webp, we need to create dir first - _ = os.MkdirAll(path.Dir(WebpAbsPath), 0755) - q, _ := strconv.ParseFloat(QUALITY, 32) - err = WebpEncoder(RawImageAbs, WebpAbsPath, float32(q), true, nil) - - if err != nil { - log.Error(err) - c.SendStatus(400) - c.Send("Bad file!") + } else { + msg := fmt.Sprintf("Remote returned %d status code!", statusCode) + c.Send(msg) + log.Warn(msg) + c.SendStatus(statusCode) + CleanProxyCache(ExhaustPath + reqURI + "*") return } - finalFile = WebpAbsPath + // End Proxy Mode + } else { + // Check the original image for existence, + if !ImageExists(RawImageAbs) { + msg := "Image not found!" + c.Send(msg) + log.Warn(msg) + c.SendStatus(404) + return + } + + _, WebpAbsPath := GenWebpAbs(RawImageAbs, ExhaustPath, ImgFilename, reqURI) + + if ImageExists(WebpAbsPath) { + finalFile = WebpAbsPath + } else { + // we don't have abc.jpg.png1582558990.webp + // delete the old pic and convert a new one. + // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp + destHalfFile := path.Clean(path.Join(WebpAbsPath, path.Dir(reqURI), ImgFilename)) + matches, err := filepath.Glob(destHalfFile + "*") + if err != nil { + log.Error(err.Error()) + } else { + // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558100.webp <- older ones will be removed + // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp <- keep the latest one + for _, p := range matches { + if strings.Compare(destHalfFile, p) != 0 { + _ = os.Remove(p) + } + } + } + + //for webp, we need to create dir first + _ = os.MkdirAll(path.Dir(WebpAbsPath), 0755) + q, _ := strconv.ParseFloat(QUALITY, 32) + err = WebpEncoder(RawImageAbs, WebpAbsPath, float32(q), true, nil) + + if err != nil { + log.Error(err) + c.SendStatus(400) + c.Send("Bad file!") + return + } + finalFile = WebpAbsPath + } + etag := GenEtag(finalFile) + c.Set("ETag", etag) + c.SendFile(finalFile) } - etag := GenEtag(finalFile) - c.Set("ETag", etag) - c.SendFile(finalFile) + } } diff --git a/webp-server.go b/webp-server.go index 1f38454..aa716ae 100644 --- a/webp-server.go +++ b/webp-server.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path" + "regexp" "runtime" "github.com/gofiber/fiber" @@ -21,7 +22,7 @@ type Config struct { ExhaustPath string `json:"EXHAUST_PATH"` } -const version = "0.1.5" +const version = "0.2.0" var configPath string var prefetch bool @@ -65,11 +66,6 @@ func loadConfig(path string) Config { defer jsonObject.Close() decoder := json.NewDecoder(jsonObject) _ = decoder.Decode(&config) - _, err = os.Stat(config.ImgPath) - if err != nil { - log.Fatalf("Your image path %s is incorrect.Please check and confirm.", config.ImgPath) - } - return config } @@ -118,7 +114,20 @@ func main() { HOST := config.HOST PORT := config.PORT - confImgPath := path.Clean(config.ImgPath) + // Check for remote address + matched, _ := regexp.MatchString(`^https?://`, config.ImgPath) + proxyMode := false + confImgPath := "" + if matched { + proxyMode = true + confImgPath = config.ImgPath + } else { + _, err := os.Stat(config.ImgPath) + if err != nil { + log.Fatalf("Your image path %s is incorrect.Please check and confirm.", config.ImgPath) + } + confImgPath = path.Clean(config.ImgPath) + } QUALITY := config.QUALITY AllowedTypes := config.AllowedTypes var ExhaustPath string @@ -141,7 +150,7 @@ func main() { // Server Info log.Infof("WebP Server %s %s", version, ListenAddress) - app.Get("/*", Convert(confImgPath, ExhaustPath, AllowedTypes, QUALITY)) + app.Get("/*", Convert(confImgPath, ExhaustPath, AllowedTypes, QUALITY, proxyMode)) app.Listen(ListenAddress) }