Feature: WebP Proxy (#51)

* feat-webp-proxy

* Fix panic on clear Cache, modified output.

* Optimize etag fetch logic

* Update README for new docs website.

* Bump version to 0.2.0.

README.md

@@ -3,7 +3,7 @@
 </p>
 <img src="https://api.travis-ci.org/webp-sh/webp_server_go.svg?branch=master"/>
 
-[Documentation](https://webp.sh/docs/) | [Website](https://webp.sh/)
+[Documentation](https://docs.webp.sh/) | [Website](https://webp.sh/)
 
 This is a Server based on Golang, which allows you to serve WebP images on the fly. 
 It will convert `jpg,jpeg,png` files by default, this can be customized by editing the `config.json`.. 
@@ -18,7 +18,7 @@ It will convert `jpg,jpeg,png` files by default, this can be customized by editi
 ## Simple Usage Steps
 
 ### 1. Download or build the binary
-Download the `webp-server` from [release](https://github.com/n0vad3v/webp_server_go/releases) page.
+Download the `webp-server` from [release](https://github.com/webp-sh/webp_server_go/releases) page.
 
 ### 2. Dump config file
 
@@ -67,8 +67,7 @@ Let Nginx to `proxy_pass http://localhost:3333/;`, and your webp-server is on-th
 
 ## Advanced Usage
 
-For supervisor, Docker sections, please read our documentation at [https://webp.sh/docs/](https://webp.sh/docs/)
+For supervisor, Docker sections, please read our documentation at [https://docs.webp.sh/](https://docs.webp.sh/)
-
 
 ## License
 

helper.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"hash/crc32"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"os"
@@ -48,6 +49,56 @@ func ImageExists(filename string) bool {
 	return !info.IsDir()
 }
 
+// Check for remote filepath, e.g: https://test.webp.sh/node.png
+// return StatusCode, etagValue
+func GetRemoteImageInfo(fileUrl string) (int, string) {
+	res, err := http.Head(fileUrl)
+	if err != nil {
+		log.Fatal("Connection to remote error!")
+	}
+	if res.StatusCode != 404 {
+		etagValue := res.Header.Get("etag")
+		if etagValue == "" {
+			log.Info("Remote didn't return etag in header, please check.")
+		} else {
+			return 200, etagValue
+		}
+	}
+	return res.StatusCode, ""
+}
+
+func FetchRemoteImage(filepath string, url string) error {
+	resp, err := http.Get(url)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	_ = os.MkdirAll(path.Dir(filepath), 0755)
+	out, err := os.Create(filepath)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	_, err = io.Copy(out, resp.Body)
+	return err
+}
+
+// Given /path/to/node.png
+// Delete /path/to/node.png*
+func CleanProxyCache(cacheImagePath string) {
+	// Delete /node.png*
+	files, err := filepath.Glob(cacheImagePath + "*")
+	if err != nil {
+		fmt.Println(err)
+	}
+	for _, f := range files {
+		if err := os.Remove(f); err != nil {
+			log.Info(err)
+		}
+	}
+}
+
 func GenWebpAbs(RawImagePath string, ExhaustPath string, ImgFilename string, reqURI string) (string, string) {
 	// get file mod time
 	STAT, err := os.Stat(RawImagePath)
@@ -56,7 +107,7 @@ func GenWebpAbs(RawImagePath string, ExhaustPath string, ImgFilename string, req
 	}
 	ModifiedTime := STAT.ModTime().Unix()
 	// webpFilename: abc.jpg.png -> abc.jpg.png1582558990.webp
-	var WebpFilename = fmt.Sprintf("%s.%d.webp", ImgFilename, ModifiedTime)
+	WebpFilename := fmt.Sprintf("%s.%d.webp", ImgFilename, ModifiedTime)
 	cwd, _ := os.Getwd()
 
 	// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp

router.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"fmt"
 	"os"
 	"path"
 	"path/filepath"
@@ -12,10 +13,10 @@ import (
 	"github.com/gofiber/fiber"
 )
 
-func Convert(ImgPath string, ExhaustPath string, AllowedTypes []string, QUALITY string) func(c *fiber.Ctx) {
+func Convert(ImgPath string, ExhaustPath string, AllowedTypes []string, QUALITY string, proxyMode bool) func(c *fiber.Ctx) {
 	return func(c *fiber.Ctx) {
 		//basic vars
-		var reqURI = c.Path()                        // mypic/123.jpg
+		var reqURI = c.Path()                        // /mypic/123.jpg
 		var RawImageAbs = path.Join(ImgPath, reqURI) // /home/xxx/mypic/123.jpg
 		var ImgFilename = path.Base(reqURI)          // pure filename, 123.jpg
 		var finalFile string                         // We'll only need one c.sendFile()
@@ -57,52 +58,102 @@ func Convert(ImgPath string, ExhaustPath string, AllowedTypes []string, QUALITY
 			return
 		}
 
-		// Check the original image for existence,
+		// Start Proxy Mode
-		if !ImageExists(RawImageAbs) {
+		if proxyMode {
-			msg := "Image not found!"
+			// https://test.webp.sh/node.png
-			c.Send(msg)
+			realRemoteAddr := ImgPath + reqURI
-			log.Warn(msg)
+			// Ping Remote for status code and etag info
-			c.SendStatus(404)
-			return
-		}
 
-		_, WebpAbsPath := GenWebpAbs(RawImageAbs, ExhaustPath, ImgFilename, reqURI)
+			// If status code is 200
-
+			//     Check for local /node.png-etag-<etagValue>
-		if ImageExists(WebpAbsPath) {
+			//         if exist
-			finalFile = WebpAbsPath
+			//             Send local cache
-		} else {
+			//         else
-			// we don't have abc.jpg.png1582558990.webp
+			//             Delete local /node.png*
-			// delete the old pic and convert a new one.
+			//             Fetch and convert to /node.png-etag-<etagValue>
-			// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
+			//             Send local cache
-			destHalfFile := path.Clean(path.Join(WebpAbsPath, path.Dir(reqURI), ImgFilename))
+			// else status code is 404
-			matches, err := filepath.Glob(destHalfFile + "*")
+			//      Delete /node.png*
-			if err != nil {
+			//      Send 404
-				log.Error(err.Error())
+			fmt.Println("Remote Addr is " + realRemoteAddr + ", fetching..")
-			} else {
+			statusCode, etagValue := GetRemoteImageInfo(realRemoteAddr)
-				// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558100.webp <- older ones will be removed
+			if statusCode == 200 {
-				// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp <- keep the latest one
+				// Check local path: /node.png-etag-<etagValue>
-				for _, p := range matches {
+				localEtagImagePath := ExhaustPath + reqURI + "-etag-" + etagValue
-					if strings.Compare(destHalfFile, p) != 0 {
+				if ImageExists(localEtagImagePath) {
-						_ = os.Remove(p)
+					c.SendFile(localEtagImagePath)
+				} else {
+					// Temporary store of remote file.
+					// ./remote-raw/node.png
+					CleanProxyCache(ExhaustPath + reqURI + "*")
+					localRemoteTmpPath := "./remote-raw" + reqURI
+					FetchRemoteImage(localRemoteTmpPath, realRemoteAddr)
+					q, _ := strconv.ParseFloat(QUALITY, 32)
+					_ = os.MkdirAll(path.Dir(localEtagImagePath), 0755)
+					err := WebpEncoder(localRemoteTmpPath, localEtagImagePath, float32(q), true, nil)
+					if err != nil {
+						fmt.Println(err)
 					}
+					c.SendFile(localEtagImagePath)
 				}
-			}
+			} else {
-
+				msg := fmt.Sprintf("Remote returned %d status code!", statusCode)
-			//for webp, we need to create dir first
+				c.Send(msg)
-			_ = os.MkdirAll(path.Dir(WebpAbsPath), 0755)
+				log.Warn(msg)
-			q, _ := strconv.ParseFloat(QUALITY, 32)
+				c.SendStatus(statusCode)
-			err = WebpEncoder(RawImageAbs, WebpAbsPath, float32(q), true, nil)
+				CleanProxyCache(ExhaustPath + reqURI + "*")
-
-			if err != nil {
-				log.Error(err)
-				c.SendStatus(400)
-				c.Send("Bad file!")
 				return
 			}
-			finalFile = WebpAbsPath
+			// End Proxy Mode
+		} else {
+			// Check the original image for existence,
+			if !ImageExists(RawImageAbs) {
+				msg := "Image not found!"
+				c.Send(msg)
+				log.Warn(msg)
+				c.SendStatus(404)
+				return
+			}
+
+			_, WebpAbsPath := GenWebpAbs(RawImageAbs, ExhaustPath, ImgFilename, reqURI)
+
+			if ImageExists(WebpAbsPath) {
+				finalFile = WebpAbsPath
+			} else {
+				// we don't have abc.jpg.png1582558990.webp
+				// delete the old pic and convert a new one.
+				// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
+				destHalfFile := path.Clean(path.Join(WebpAbsPath, path.Dir(reqURI), ImgFilename))
+				matches, err := filepath.Glob(destHalfFile + "*")
+				if err != nil {
+					log.Error(err.Error())
+				} else {
+					// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558100.webp <- older ones will be removed
+					// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp <- keep the latest one
+					for _, p := range matches {
+						if strings.Compare(destHalfFile, p) != 0 {
+							_ = os.Remove(p)
+						}
+					}
+				}
+
+				//for webp, we need to create dir first
+				_ = os.MkdirAll(path.Dir(WebpAbsPath), 0755)
+				q, _ := strconv.ParseFloat(QUALITY, 32)
+				err = WebpEncoder(RawImageAbs, WebpAbsPath, float32(q), true, nil)
+
+				if err != nil {
+					log.Error(err)
+					c.SendStatus(400)
+					c.Send("Bad file!")
+					return
+				}
+				finalFile = WebpAbsPath
+			}
+			etag := GenEtag(finalFile)
+			c.Set("ETag", etag)
+			c.SendFile(finalFile)
 		}
-		etag := GenEtag(finalFile)
+
-		c.Set("ETag", etag)
-		c.SendFile(finalFile)
 	}
 }

webp-server.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"regexp"
 	"runtime"
 
 	"github.com/gofiber/fiber"
@@ -21,7 +22,7 @@ type Config struct {
 	ExhaustPath  string   `json:"EXHAUST_PATH"`
 }
 
-const version = "0.1.5"
+const version = "0.2.0"
 
 var configPath string
 var prefetch bool
@@ -65,11 +66,6 @@ func loadConfig(path string) Config {
 	defer jsonObject.Close()
 	decoder := json.NewDecoder(jsonObject)
 	_ = decoder.Decode(&config)
-	_, err = os.Stat(config.ImgPath)
-	if err != nil {
-		log.Fatalf("Your image path %s is incorrect.Please check and confirm.", config.ImgPath)
-	}
-
 	return config
 }
 
@@ -118,7 +114,20 @@ func main() {
 
 	HOST := config.HOST
 	PORT := config.PORT
-	confImgPath := path.Clean(config.ImgPath)
+	// Check for remote address
+	matched, _ := regexp.MatchString(`^https?://`, config.ImgPath)
+	proxyMode := false
+	confImgPath := ""
+	if matched {
+		proxyMode = true
+		confImgPath = config.ImgPath
+	} else {
+		_, err := os.Stat(config.ImgPath)
+		if err != nil {
+			log.Fatalf("Your image path %s is incorrect.Please check and confirm.", config.ImgPath)
+		}
+		confImgPath = path.Clean(config.ImgPath)
+	}
 	QUALITY := config.QUALITY
 	AllowedTypes := config.AllowedTypes
 	var ExhaustPath string
@@ -141,7 +150,7 @@ func main() {
 	// Server Info
 	log.Infof("WebP Server %s %s", version, ListenAddress)
 
-	app.Get("/*", Convert(confImgPath, ExhaustPath, AllowedTypes, QUALITY))
+	app.Get("/*", Convert(confImgPath, ExhaustPath, AllowedTypes, QUALITY, proxyMode))
 	app.Listen(ListenAddress)
 
 }