prefetch is finish. resolve #5.

Waiting for nova to test and merge to master

pics/dir1/inside.jpg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6a4905ad370e77394651983edfeb7f5d52c6576d5f7bd28da746afaf0197f3f
+size 210536

webp-server.go

@@ -1,12 +1,12 @@
 package main
 
 import (
+	"bufio"
 	"bytes"
 	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
-	"github.com/gofiber/fiber"
 	"image"
 	"image/jpeg"
 	"image/png"
@@ -20,6 +20,7 @@ import (
 	"strings"
 
 	"github.com/chai2010/webp"
+	"github.com/gofiber/fiber"
 )
 
 type Config struct {
@@ -31,6 +32,7 @@ type Config struct {
 }
 
 var configPath string
+var prefetch bool
 
 func loadConfig(path string) Config {
 	var config Config
@@ -52,15 +54,6 @@ func imageExists(filename string) bool {
 	return !info.IsDir()
 }
 
-func Find(slice []string, val string) (int, bool) {
-	for i, item := range slice {
-		if item == val {
-			return i, true
-		}
-	}
-	return -1, false
-}
-
 func GetFileContentType(buffer []byte) string {
 	// Use the net/http package's handy DectectContentType function. Always returns a valid
 	// content-type by returning "application/octet-stream" if no others seemed to match.
@@ -68,7 +61,7 @@ func GetFileContentType(buffer []byte) string {
 	return contentType
 }
 
-func webpEncoder(p1, p2 string, quality float32) (err error) {
+func webpEncoder(p1, p2 string, quality float32, Log bool) (err error) {
 	// if convert fails, return error; success nil
 	var buf bytes.Buffer
 	var img image.Image
@@ -83,10 +76,14 @@ func webpEncoder(p1, p2 string, quality float32) (err error) {
 	} else if strings.Contains(contentType, "png") {
 		img, _ = png.Decode(bytes.NewReader(data))
 	}
+	// TODO should we add bmp and tiff support? Need more packages.
+	// import "golang.org/x/image/bmp"
+	// import "golang.org/x/image/tiff"
 
 	if img == nil {
-		log.Println("Image file is corrupted or not supported!")
+		msg := "image file " + path.Base(p1) + " is corrupted or not supported"
-		err = errors.New("image file is corrupted or not supported")
+		log.Println(msg)
+		err = errors.New(msg)
 		return
 	}
 
@@ -94,128 +91,85 @@ func webpEncoder(p1, p2 string, quality float32) (err error) {
 		log.Println(err)
 		return
 	}
-	if err = ioutil.WriteFile(p2, buf.Bytes(), 0666); err != nil {
+	if err = ioutil.WriteFile(p2, buf.Bytes(), os.ModePerm); err != nil {
 		log.Println(err)
 		return
 	}
 
-	fmt.Println("Save to webp ok")
+	if Log {
+		fmt.Printf("Save to %s ok\n", p2)
+	}
 	return nil
 }
 
 func init() {
 	flag.StringVar(&configPath, "config", "config.json", "/path/to/config.json. (Default: ./config.json)")
+	flag.BoolVar(&prefetch, "prefetch", false, "Prefetch and convert image to webp")
 	flag.Parse()
 }
 
-func main() {
+func Convert(ImgPath string, AllowedTypes []string, QUALITY string) func(c *fiber.Ctx) {
-	app := fiber.New()
+	return func(c *fiber.Ctx) {
-	app.Banner = false
+		//basic vars
-	app.Server = "WebP Server Go"
+		var reqURI = c.Path()                        // mypic/123.jpg
-
+		var RawImageAbs = path.Join(ImgPath, reqURI) // /home/xxx/mypic/123.jpg
-	config := loadConfig(configPath)
+		var ImgFilename = path.Base(reqURI)          // pure filename, 123.jpg
-
+		var finalFile string                         // We'll only need one c.sendFile()
-	HOST := config.HOST
+		// Check for Safari users. If they're Safari, just simply ignore everything.
-	PORT := config.PORT
-	ImgPath := config.ImgPath
-	QUALITY := config.QUALITY
-	AllowedTypes := config.AllowedTypes
-
-	ListenAddress := HOST + ":" + PORT
-
-	// Server Info
-	ServerInfo := "WebP Server is running at " + ListenAddress
-	fmt.Println(ServerInfo)
-
-	app.Get("/*", func(c *fiber.Ctx) {
-
-		// /var/www/IMG_PATH/path/to/tsuki.jpg
-		ImgAbsolutePath := ImgPath + c.Path()
-
-		// /path/to/tsuki.jpg
-		ImgPath := c.Path()
-
-		// jpg
-		seps := strings.Split(path.Ext(ImgPath), ".")
-		var ImgExt string
-		if len(seps) >= 2 {
-			ImgExt = seps[1]
-		} else {
-			c.Send("Invalid request")
-			return
-		}
-
-		// tsuki.jpg
-		ImgName := path.Base(ImgPath)
-
-		// /path/to
-		DirPath := path.Dir(ImgPath)
-
-		// Check the original image for existence
-		OriginalImgExists := imageExists(ImgAbsolutePath)
-		if !OriginalImgExists {
-			c.Send("File not found!")
-			c.SendStatus(404)
-			return
-		}
-
-		// 1582558990
-		STAT, err := os.Stat(ImgAbsolutePath)
-		if err != nil {
-			fmt.Println(err.Error())
-		}
-		ModifiedTime := STAT.ModTime().Unix()
-
-		// /path/to/tsuki.jpg.1582558990.webp
-		WebpImgPath := fmt.Sprintf("%s/%s.%d.webp", DirPath, ImgName, ModifiedTime)
-
-		// /home/webp_server
-		CurrentPath, err := os.Getwd()
-		if err != nil {
-			fmt.Println(err.Error())
-		}
-
-		// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
-		WebpAbsolutePath := path.Clean(CurrentPath + "/exhaust" + WebpImgPath)
-
-		// /home/webp_server/exhaust/path/to
-		DirAbsolutePath := path.Clean(CurrentPath + "/exhaust" + DirPath)
-
-		// Check file extension
-		_, found := Find(AllowedTypes, ImgExt)
-		if !found {
-			c.Send("File extension not allowed!")
-			c.SendStatus(403)
-			return
-		}
-
-		// Check the original image for existence
-		if !OriginalImgExists {
-			// The original image doesn't exist, check the webp image, delete if processed.
-			if imageExists(WebpAbsolutePath) {
-				_ = os.Remove(WebpAbsolutePath)
-			}
-			c.Send("File not found!")
-			c.SendStatus(404)
-			return
-		}
-
-		// Check for Safari users
 		UA := c.Get("User-Agent")
-		if strings.Contains(UA, "Safari") && !strings.Contains(UA, "Chrome") && !strings.Contains(UA, "Firefox") {
+		if strings.Contains(UA, "Safari") && !strings.Contains(UA, "Chrome") &&
-			c.SendFile(ImgAbsolutePath)
+			!strings.Contains(UA, "Firefox") {
+			finalFile = RawImageAbs
 			return
 		}
 
-		if imageExists(WebpAbsolutePath) {
+		// check ext
-			c.SendFile(WebpAbsolutePath)
+		// TODO: should remove this function. Check in Nginx.
-		} else {
+		for _, ext := range AllowedTypes {
-			// Mkdir
+			haystack := strings.ToLower(ImgFilename)
-			_ = os.MkdirAll(DirAbsolutePath, os.ModePerm)
+			needle := strings.ToLower("." + ext)
+			if strings.HasSuffix(haystack, needle) {
+				break
+			} else {
+				c.Send("File extension not allowed!")
+				c.SendStatus(403)
+				return
+			}
+		}
 
-			// cwebp -q 60 Cute-Baby-Girl.png -o Cute-Baby-Girl.webp
+		// Check the original image for existence,
+		if !imageExists(RawImageAbs) {
+			c.Send("Image not found!")
+			c.SendStatus(404)
+			return
+		}
+
+		cwd, WebpAbsPath := genWebpAbs(RawImageAbs, ImgFilename, reqURI)
+
+		if imageExists(WebpAbsPath) {
+			finalFile = WebpAbsPath
+		} else {
+			// we don't have abc.jpg.png1582558990.webp
+			// delete the old pic and convert a new one.
+			// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
+			destHalfFile := path.Clean(path.Join(cwd, "exhaust", path.Dir(reqURI), ImgFilename))
+			matches, err := filepath.Glob(destHalfFile + "*")
+			if err != nil {
+				fmt.Println(err.Error())
+			} else {
+				// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558100.webp <- older ones will be removed
+				// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp <- keep the latest one
+				for _, p := range matches {
+					if strings.Compare(destHalfFile, p) != 0 {
+						_ = os.Remove(p)
+					}
+				}
+			}
+
+			//for webp, we need to create dir first
+			_ = os.MkdirAll(path.Dir(WebpAbsPath), os.ModePerm)
 			q, _ := strconv.ParseFloat(QUALITY, 32)
-			err = webpEncoder(ImgAbsolutePath, WebpAbsolutePath, float32(q))
+			err = webpEncoder(RawImageAbs, WebpAbsPath, float32(q), true)
 
 			if err != nil {
 				fmt.Println(err)
@@ -223,26 +177,95 @@ func main() {
 				c.Send("Bad file!")
 				return
 			}
-
+			finalFile = WebpAbsPath
-			ImgNameCopy := string([]byte(ImgName))
-			c.SendFile(WebpAbsolutePath)
-
-			// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558100.webp <- older ones will be removed
-			// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp <- keep the latest one
-			WebpCachedImgPath := path.Clean(fmt.Sprintf("%s/exhaust%s/%s.*.webp", CurrentPath, DirPath, ImgNameCopy))
-			matches, err := filepath.Glob(WebpCachedImgPath)
-			if err != nil {
-				fmt.Println(err.Error())
-			} else {
-				for _, p := range matches {
-					if strings.Compare(WebpAbsolutePath, p) != 0 {
-						_ = os.Remove(p)
-					}
-				}
-			}
 		}
-	})
+		c.SendFile(finalFile)
+	}
+}
 
+func fileCount(dir string) int {
+	count := 0
+	_ = filepath.Walk(dir,
+		func(path string, info os.FileInfo, err error) error {
+			if !info.IsDir() {
+				count += 1
+			}
+			return nil
+		})
+	return count
+}
+
+func genWebpAbs(RawImagePath string, ImgFilename string, reqURI string) (string, string) {
+	// get file mod time
+	STAT, err := os.Stat(RawImagePath)
+	if err != nil {
+		fmt.Println(err.Error())
+	}
+	ModifiedTime := STAT.ModTime().Unix()
+	// webpFilename: abc.jpg.png -> abc.jpg.png1582558990.webp
+	var WebpFilename = fmt.Sprintf("%s.%d.webp", ImgFilename, ModifiedTime)
+	cwd, _ := os.Getwd()
+
+	// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
+	WebpAbsolutePath := path.Clean(path.Join(cwd, "exhaust", path.Dir(reqURI), WebpFilename))
+	return cwd, WebpAbsolutePath
+}
+
+func prefetchImages(confImgPath string, QUALITY string) {
+	fmt.Println(`Prefetch will convert all your images to webp, it may take some time and consume a lot of CPU resource. Do you want to proceed(Y/n)`)
+	reader := bufio.NewReader(os.Stdin)
+	char, _, _ := reader.ReadRune() //y Y enter
+	if char == 121 || char == 10 || char == 89 {
+		//prefetch, recursive through the dir
+		all := fileCount(confImgPath)
+		count := 0
+		err := filepath.Walk(confImgPath,
+			func(picAbsPath string, info os.FileInfo, err error) error {
+				if err != nil {
+					return err
+				}
+				// RawImagePath string, ImgFilename string, reqURI string
+				proposedURI := strings.Replace(picAbsPath, confImgPath, "", 1)
+				_, p2 := genWebpAbs(picAbsPath, info.Name(), proposedURI)
+				q, _ := strconv.ParseFloat(QUALITY, 32)
+				_ = os.MkdirAll(path.Dir(p2), os.ModePerm)
+				_ = webpEncoder(picAbsPath, p2, float32(q), false)
+				count += 1
+				// progress bar
+				_, _ = fmt.Fprintf(os.Stdout, "Convert in progress: %d/%d\r", count, all)
+				return nil
+			})
+		if err != nil {
+			log.Println(err)
+		}
+	}
+	fmt.Fprintf(os.Stdout, "Prefetch complete,")
+}
+
+func main() {
+	config := loadConfig(configPath)
+
+	HOST := config.HOST
+	PORT := config.PORT
+	confImgPath := path.Clean(config.ImgPath)
+	QUALITY := config.QUALITY
+	AllowedTypes := config.AllowedTypes
+
+	if prefetch {
+		prefetchImages(confImgPath, QUALITY)
+	}
+
+	app := fiber.New()
+	app.Banner = false
+	app.Server = "WebP Server Go"
+
+	ListenAddress := HOST + ":" + PORT
+
+	// Server Info
+	ServerInfo := "WebP Server is running at " + ListenAddress
+	fmt.Println(ServerInfo)
+
+	app.Get("/*", Convert(confImgPath, AllowedTypes, QUALITY))
 	app.Listen(ListenAddress)
 
 }