Fiber v2, code format, typo,test case, go 1.15 and more (#54)

* upgrade to fiber v2

* code format

* remove redundant variables
* remove useless exportable variables/functions

* go mod replace

use our own mirror now.

* add test case for converter, use deferInit to make test more simple

* remove useless file and fix typo

* Makefile change
* upgrade to go 1.15
* remove wrong go test comments

* complete test case, coverage, coverage badge

* Fix version typo

* config struct fix

* add banner, show version, add server header, remove fiber startup message

Co-authored-by: n0vad3v <n0vad3v@riseup.net>

.travis.yml

@@ -1,7 +1,7 @@
 language: go
 
-go: 
+go:
- - 1.14.7
+  - 1.15.4
 
 env: GO111MODULE=on
 arch:
@@ -22,8 +22,7 @@ jobs:
       os:
         - linux
       script:
-        - go test -v -cover encoder_test.go encoder.go helper.go
+        - make test
-        - go test -v -cover helper_test.go helper.go
 
 deploy:
   provider: releases
@@ -35,3 +34,6 @@ deploy:
       repo: webp-sh/webp_server_go
       tags: true
       branch: master
+
+after_success:
+  - bash <(curl -s https://codecov.io/bash)

Makefile

@@ -10,6 +10,17 @@ else
 	ARCH=amd64
 endif
 
-all: build
+default:
-build: 
+	make clean
-	./scripts/build.sh $(OS) $(ARCH)
+	go build -o builds/webp-server-$(OS)-$(ARCH) .
+	ls builds
+all:
+	make clean
+	./scripts/build.sh $(OS) $(ARCH)
+
+test:
+	go test -coverprofile=coverage.txt -covermode=atomic
+
+clean:
+	rm -rf builds
+	rm -rf prefetch

README.md

@@ -3,6 +3,8 @@
 </p>
 <img src="https://api.travis-ci.org/webp-sh/webp_server_go.svg?branch=master"/>
 
+[![codecov](https://codecov.io/gh/webp-sh/webp_server_go/branch/master/graph/badge.svg?token=VR3BMZME65)](https://codecov.io/gh/webp-sh/webp_server_go)
+
 [Documentation](https://docs.webp.sh/) | [Website](https://webp.sh/)
 
 This is a Server based on Golang, which allows you to serve WebP images on the fly. 

config.json

@@ -3,6 +3,11 @@
   "PORT": "3333",
   "QUALITY": "80",
   "IMG_PATH": "./pics",
-  "EXHAUST_PATH": "",
+  "EXHAUST_PATH": "./exhaust",
-  "ALLOWED_TYPES": ["jpg","png","jpeg","bmp"]
+  "ALLOWED_TYPES": [
+    "jpg",
+    "png",
+    "jpeg",
+    "bmp"
+  ]
 }

encoder.go

@@ -17,7 +17,7 @@ import (
 	"golang.org/x/image/bmp"
 )
 
-func WebpEncoder(p1, p2 string, quality float32, Log bool, c chan int) (err error) {
+func webpEncoder(p1, p2 string, quality float32, Log bool, c chan int) (err error) {
 	// if convert fails, return error; success nil
 
 	log.Debugf("target: %s with quality of %f", path.Base(p1), quality)
@@ -26,11 +26,11 @@ func WebpEncoder(p1, p2 string, quality float32, Log bool, c chan int) (err erro
 
 	data, err := ioutil.ReadFile(p1)
 	if err != nil {
-		ChanErr(c)
+		chanErr(c)
 		return
 	}
 
-	contentType := GetFileContentType(data[:512])
+	contentType := getFileContentType(data[:512])
 	if strings.Contains(contentType, "jpeg") {
 		img, _ = jpeg.Decode(bytes.NewReader(data))
 	} else if strings.Contains(contentType, "png") {
@@ -47,18 +47,18 @@ func WebpEncoder(p1, p2 string, quality float32, Log bool, c chan int) (err erro
 		msg := "image file " + path.Base(p1) + " is corrupted or not supported"
 		log.Debug(msg)
 		err = errors.New(msg)
-		ChanErr(c)
+		chanErr(c)
 		return
 	}
 
 	if err = webp.Encode(&buf, img, &webp.Options{Lossless: false, Quality: quality}); err != nil {
 		log.Error(err)
-		ChanErr(c)
+		chanErr(c)
 		return
 	}
 	if err = ioutil.WriteFile(p2, buf.Bytes(), 0644); err != nil {
 		log.Error(err)
-		ChanErr(c)
+		chanErr(c)
 		return
 	}
 
@@ -66,7 +66,7 @@ func WebpEncoder(p1, p2 string, quality float32, Log bool, c chan int) (err erro
 		log.Info("Save to " + p2 + " ok!\n")
 	}
 
-	ChanErr(c)
+	chanErr(c)
 
 	return nil
 }

encoder_test.go

@@ -1,15 +1,15 @@
 package main
 
 import (
+	"github.com/stretchr/testify/assert"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"testing"
 )
 
-//go test -v -cover encoder_test.go encoder.go helper.go
+//go test -v -cover .
 func TestWebpEncoder(t *testing.T) {
-
 	var webp = "/tmp/test-result.webp"
 	var target = walker()
 
@@ -19,8 +19,24 @@ func TestWebpEncoder(t *testing.T) {
 	}
 	_ = os.Remove(webp)
 
+	// test error
+	err := webpEncoder("./pics/empty.jpg", webp, 80, true, nil)
+	assert.NotNil(t, err)
 }
 
+func TestNonImage(t *testing.T) {
+	var webp = "/tmp/test-result.webp"
+	// test error
+	var err = webpEncoder("./pics/empty.jpg", webp, 80, true, nil)
+	assert.NotNil(t, err)
+}
+
+func TestWriteFail(t *testing.T) {
+	// test permission denied
+	var webp = "/123.webp"
+	var err = webpEncoder("./pics/png.jpg", webp, 80, true, nil)
+	assert.NotNil(t, err)
+}
 func walker() []string {
 	var list []string
 	_ = filepath.Walk("./pics", func(path string, info os.FileInfo, err error) error {
@@ -34,8 +50,8 @@ func walker() []string {
 
 func runEncoder(t *testing.T, file string, webp string) {
 	var c chan int
-	//t.Logf("Convert from %s to %s", file, webp)
+	//t.Logf("convert from %s to %s", file, webp)
-	var err = WebpEncoder(file, webp, 80, false, c)
+	var err = webpEncoder(file, webp, 80, true, c)
 	if file == "pics/empty.jpg" && err != nil {
 		t.Log("Empty file, that's okay.")
 	} else if err != nil {
@@ -43,7 +59,7 @@ func runEncoder(t *testing.T, file string, webp string) {
 	}
 
 	data, _ := ioutil.ReadFile(webp)
-	types := GetFileContentType(data[:512])
+	types := getFileContentType(data[:512])
 	if types != "image/webp" {
 		t.Fatal("Fatal, file type is wrong!")
 	}

go.mod

@@ -1,11 +1,16 @@
 module webp_server_go
 
-go 1.13
+go 1.15
 
 require (
 	github.com/chai2010/webp v1.1.0
-	github.com/gofiber/fiber v1.4.0
+	github.com/gofiber/fiber/v2 v2.1.4
 	github.com/sirupsen/logrus v1.6.0
 	github.com/stretchr/testify v1.3.0
 	golang.org/x/image v0.0.0-20200119044424-58c23975cae1
 )
+
+replace (
+	github.com/gofiber/fiber/v2 v2.1.4  => github.com/webp-sh/fiber/v2 v2.1.4
+	github.com/chai2010/webp v1.1.0  => github.com/webp-sh/webp v1.1.1
+)

helper.go

@@ -15,20 +15,20 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-func ChanErr(ccc chan int) {
+func chanErr(ccc chan int) {
 	if ccc != nil {
 		ccc <- 1
 	}
 }
 
-func GetFileContentType(buffer []byte) string {
+func getFileContentType(buffer []byte) string {
 	// Use the net/http package's handy DectectContentType function. Always returns a valid
 	// content-type by returning "application/octet-stream" if no others seemed to match.
 	contentType := http.DetectContentType(buffer)
 	return contentType
 }
 
-func FileCount(dir string) int {
+func fileCount(dir string) int {
 	count := 0
 	_ = filepath.Walk(dir,
 		func(path string, info os.FileInfo, err error) error {
@@ -40,7 +40,7 @@ func FileCount(dir string) int {
 	return count
 }
 
-func ImageExists(filename string) bool {
+func imageExists(filename string) bool {
 	info, err := os.Stat(filename)
 	if os.IsNotExist(err) {
 		return false
@@ -51,10 +51,11 @@ func ImageExists(filename string) bool {
 
 // Check for remote filepath, e.g: https://test.webp.sh/node.png
 // return StatusCode, etagValue
-func GetRemoteImageInfo(fileUrl string) (int, string) {
+func getRemoteImageInfo(fileUrl string) (int, string) {
 	res, err := http.Head(fileUrl)
 	if err != nil {
-		log.Fatal("Connection to remote error!")
+		log.Errorln("Connection to remote error!")
+		return http.StatusInternalServerError, ""
 	}
 	if res.StatusCode != 404 {
 		etagValue := res.Header.Get("etag")
@@ -67,7 +68,7 @@ func GetRemoteImageInfo(fileUrl string) (int, string) {
 	return res.StatusCode, ""
 }
 
-func FetchRemoteImage(filepath string, url string) error {
+func fetchRemoteImage(filepath string, url string) error {
 	resp, err := http.Get(url)
 	if err != nil {
 		return err
@@ -86,7 +87,7 @@ func FetchRemoteImage(filepath string, url string) error {
 
 // Given /path/to/node.png
 // Delete /path/to/node.png*
-func CleanProxyCache(cacheImagePath string) {
+func cleanProxyCache(cacheImagePath string) {
 	// Delete /node.png*
 	files, err := filepath.Glob(cacheImagePath + "*")
 	if err != nil {
@@ -99,11 +100,12 @@ func CleanProxyCache(cacheImagePath string) {
 	}
 }
 
-func GenWebpAbs(RawImagePath string, ExhaustPath string, ImgFilename string, reqURI string) (string, string) {
+func genWebpAbs(RawImagePath string, ExhaustPath string, ImgFilename string, reqURI string) (string, string) {
 	// get file mod time
 	STAT, err := os.Stat(RawImagePath)
 	if err != nil {
 		log.Error(err.Error())
+		return "", ""
 	}
 	ModifiedTime := STAT.ModTime().Unix()
 	// webpFilename: abc.jpg.png -> abc.jpg.png1582558990.webp
@@ -116,7 +118,7 @@ func GenWebpAbs(RawImagePath string, ExhaustPath string, ImgFilename string, req
 	return cwd, WebpAbsolutePath
 }
 
-func GenEtag(ImgAbsPath string) string {
+func genEtag(ImgAbsPath string) string {
 	data, err := ioutil.ReadFile(ImgAbsPath)
 	if err != nil {
 		log.Info(err)

helper_test.go

@@ -1,48 +1,49 @@
 package main
 
 import (
+	"io/ioutil"
+	"net/http"
+	"path/filepath"
 	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
-// test this file: go test -v -cover helper_test.go helper.go
+// test this file: go test -v -cover .
-// test one function: go test -run TestGetFileContentType helper_test.go helper.go -v
 func TestGetFileContentType(t *testing.T) {
 	var data = []byte("hello")
 	var expected = "text/plain; charset=utf-8"
-	var result = GetFileContentType(data)
+	var result = getFileContentType(data)
 
 	assert.Equalf(t, result, expected, "Result: [%s], Expected: [%s]", result, expected)
 
 }
 
-// TODO: make a universal logging function
 func TestFileCount(t *testing.T) {
 	var data = ".github"
 	var expected = 2
-	var result = FileCount(data)
+	var result = fileCount(data)
 	assert.Equalf(t, result, expected, "Result: [%d], Expected: [%d]", result, expected)
 
 }
 
 func TestImageExists(t *testing.T) {
 	var data = "./pics/empty.jpg"
-	var result = !ImageExists(data)
+	var result = !imageExists(data)
 
 	if result {
 		t.Errorf("Result: [%v], Expected: [%v]", result, false)
 	}
 	data = ".pics/empty2.jpg"
-	result = ImageExists(data)
+	result = imageExists(data)
 
 	assert.Falsef(t, result, "Result: [%v], Expected: [%v]", result, false)
 
 }
 
 func TestGenWebpAbs(t *testing.T) {
-	cwd, cooked := GenWebpAbs("./pics/webp_server.png", "/tmp",
+	cwd, cooked := genWebpAbs("./pics/webp_server.png", "/tmp",
 		"test", "a")
 	if !strings.Contains(cwd, "webp_server_go") {
 		t.Logf("Result: [%v], Expected: [%v]", cwd, "webp_server_go")
@@ -56,7 +57,7 @@ func TestGenWebpAbs(t *testing.T) {
 func TestGenEtag(t *testing.T) {
 	var data = "./pics/png.jpg"
 	var expected = "W/\"1020764-262C0329\""
-	var result = GenEtag(data)
+	var result = genEtag(data)
 
 	assert.Equalf(t, result, expected, "Result: [%s], Expected: [%s]", result, expected)
 
@@ -104,3 +105,55 @@ func TestGoOrigin(t *testing.T) {
 	}
 
 }
+
+func TestChanErr(t *testing.T) {
+	var value = 2
+	var testC = make(chan int, 2)
+	testC <- value
+	chanErr(testC)
+	value = <-testC
+	assert.Equal(t, 2, value)
+}
+
+func TestGetRemoteImageInfo(t *testing.T) {
+	url := "http://github.com/favicon.ico"
+	statusCode, etag := getRemoteImageInfo(url)
+	assert.NotEqual(t, "", etag)
+	assert.Equal(t, statusCode, http.StatusOK)
+
+	// test non-exist url
+	url = "http://sdahjajda.com"
+	statusCode, etag = getRemoteImageInfo(url)
+	assert.Equal(t, "", etag)
+	assert.Equal(t, statusCode, http.StatusInternalServerError)
+}
+
+func TestFetchRemoteImage(t *testing.T) {
+	// test the normal one
+	fp := filepath.Join("./exhaust", "test.ico")
+	url := "http://github.com/favicon.ico"
+	err := fetchRemoteImage(fp, url)
+	assert.Equal(t, err, nil)
+	data, _ := ioutil.ReadFile(fp)
+	assert.Equal(t, "image/x-icon", getFileContentType(data))
+
+	// test can't create file
+	err = fetchRemoteImage("/", url)
+	assert.NotNil(t, err)
+
+	// test bad url
+	err = fetchRemoteImage(fp, "http://ahjdsgdsghja.cya")
+	assert.NotNil(t, err)
+}
+
+func TestCleanProxyCache(t *testing.T) {
+	// test normal situation
+	fp := filepath.Join("./exhaust", "sample.png.12345.webp")
+	_ = ioutil.WriteFile(fp, []byte("1234"), 0755)
+	assert.True(t, imageExists(fp))
+	cleanProxyCache(fp)
+	assert.False(t, imageExists(fp))
+
+	// test bad dir
+	cleanProxyCache("/aasdyg/dhj2/dagh")
+}

prefetch.go

@@ -4,15 +4,15 @@ import (
 	"fmt"
 	"os"
 	"path"
-	"time"
 	"path/filepath"
 	"strconv"
 	"strings"
+	"time"
 
 	log "github.com/sirupsen/logrus"
 )
 
-func PrefetchImages(confImgPath string, ExhaustPath string, QUALITY string) {
+func prefetchImages(confImgPath string, ExhaustPath string, QUALITY string) {
 	var sTime = time.Now()
 	// maximum ongoing prefetch is depending on your core of CPU
 	log.Infof("Prefetching using %d cores", jobs)
@@ -22,7 +22,7 @@ func PrefetchImages(confImgPath string, ExhaustPath string, QUALITY string) {
 	}
 
 	//prefetch, recursive through the dir
-	all := FileCount(confImgPath)
+	all := fileCount(confImgPath)
 	count := 0
 	err := filepath.Walk(confImgPath,
 		func(picAbsPath string, info os.FileInfo, err error) error {
@@ -31,10 +31,10 @@ func PrefetchImages(confImgPath string, ExhaustPath string, QUALITY string) {
 			}
 			// RawImagePath string, ImgFilename string, reqURI string
 			proposedURI := strings.Replace(picAbsPath, confImgPath, "", 1)
-			_, p2 := GenWebpAbs(picAbsPath, ExhaustPath, info.Name(), proposedURI)
+			_, p2 := genWebpAbs(picAbsPath, ExhaustPath, info.Name(), proposedURI)
 			q, _ := strconv.ParseFloat(QUALITY, 32)
 			_ = os.MkdirAll(path.Dir(p2), 0755)
-			go WebpEncoder(picAbsPath, p2, float32(q), false, finishChan)
+			go webpEncoder(picAbsPath, p2, float32(q), false, finishChan)
 			count += <-finishChan
 			//progress bar
 			_, _ = fmt.Fprintf(os.Stdout, "[Webp Server started] - convert in progress: %d/%d\r", count, all)
@@ -45,6 +45,6 @@ func PrefetchImages(confImgPath string, ExhaustPath string, QUALITY string) {
 	}
 	elapsed := time.Since(sTime)
 	_, _ = fmt.Fprintf(os.Stdout, "Prefetch completeY(^_^)Y\n\n")
-	_, _ = fmt.Fprintf(os.Stdout, "Convert %d file in %s (^_^)Y\n\n", count, elapsed)
+	_, _ = fmt.Fprintf(os.Stdout, "convert %d file in %s (^_^)Y\n\n", count, elapsed)
 
 }

prefetch_test.go

@@ -0,0 +1,29 @@
+// webp_server_go - prefetch_test.go
+// 2020-11-10 09:27
+// Benny <benny.think@gmail.com>
+
+package main
+
+import (
+	"github.com/stretchr/testify/assert"
+	"os"
+	"testing"
+)
+
+func TestPrefetchImages(t *testing.T) {
+	// single thread
+	fp := "./prefetch"
+	_ = os.Mkdir(fp, 0755)
+	prefetchImages("./pics", "./prefetch", "80")
+	count := fileCount("./prefetch")
+	assert.Equal(t, 6, count)
+	_ = os.RemoveAll(fp)
+
+	// concurrency
+	jobs = 2
+	_ = os.Mkdir(fp, 0755)
+	prefetchImages("./pics", "./prefetch", "80")
+	count = fileCount("./prefetch")
+	assert.Equal(t, 4, count)
+	_ = os.RemoveAll(fp)
+}

router.go

@@ -1,159 +1,141 @@
 package main
 
 import (
+	"errors"
 	"fmt"
+	"github.com/gofiber/fiber/v2"
+	log "github.com/sirupsen/logrus"
 	"os"
 	"path"
 	"path/filepath"
 	"strconv"
 	"strings"
-
-	log "github.com/sirupsen/logrus"
-
-	"github.com/gofiber/fiber"
 )
 
-func Convert(ImgPath string, ExhaustPath string, AllowedTypes []string, QUALITY string, proxyMode bool) func(c *fiber.Ctx) {
+func convert(c *fiber.Ctx) error {
-	return func(c *fiber.Ctx) {
+	//basic vars
-		//basic vars
+	var reqURI = c.Path()                               // /mypic/123.jpg
-		var reqURI = c.Path()                        // /mypic/123.jpg
+	var rawImageAbs = path.Join(config.ImgPath, reqURI) // /home/xxx/mypic/123.jpg
-		var RawImageAbs = path.Join(ImgPath, reqURI) // /home/xxx/mypic/123.jpg
+	var imgFilename = path.Base(reqURI)                 // pure filename, 123.jpg
-		var ImgFilename = path.Base(reqURI)          // pure filename, 123.jpg
+	var finalFile string                                // We'll only need one c.sendFile()
-		var finalFile string                         // We'll only need one c.sendFile()
+	var UA = c.Get("User-Agent")
-		var UA = c.Get("User-Agent")
+	done := goOrigin(UA)
-		done := goOrigin(UA)
+	if done {
-		if done {
+		log.Infof("A Safari/IE/whatever user has arrived...%s", UA)
-			log.Infof("A Safari/IE/whatever user has arrived...%s", UA)
+		// Check for Safari users. If they're Safari, just simply ignore everything.
-			// Check for Safari users. If they're Safari, just simply ignore everything.
 
-			etag := GenEtag(RawImageAbs)
+		etag := genEtag(rawImageAbs)
-			c.Set("ETag", etag)
+		c.Set("ETag", etag)
-			c.SendFile(RawImageAbs)
+		return c.SendFile(rawImageAbs)
-			return
+	}
-		}
+	log.Debugf("Incoming connection from %s@%s with %s", UA, c.IP(), imgFilename)
-		log.Debugf("Incoming connection from %s@%s with %s", UA, c.IP(), ImgFilename)
 
-		// check ext
+	// check ext
-		// TODO: may remove this function. Check in Nginx.
+	var allowed = false
-		var allowed = false
+	for _, ext := range config.AllowedTypes {
-		for _, ext := range AllowedTypes {
+		haystack := strings.ToLower(imgFilename)
-			haystack := strings.ToLower(ImgFilename)
+		needle := strings.ToLower("." + ext)
-			needle := strings.ToLower("." + ext)
+		if strings.HasSuffix(haystack, needle) {
-			if strings.HasSuffix(haystack, needle) {
+			allowed = true
-				allowed = true
+			break
-				break
-			} else {
-				allowed = false
-			}
-		}
-		if !allowed {
-			msg := "File extension not allowed! " + ImgFilename
-			log.Warn(msg)
-			c.Send(msg)
-			if ImageExists(RawImageAbs) {
-				etag := GenEtag(RawImageAbs)
-				c.Set("ETag", etag)
-				c.SendFile(RawImageAbs)
-			}
-			return
-		}
-
-		// Start Proxy Mode
-		if proxyMode {
-			// https://test.webp.sh/node.png
-			realRemoteAddr := ImgPath + reqURI
-			// Ping Remote for status code and etag info
-
-			// If status code is 200
-			//     Check for local /node.png-etag-<etagValue>
-			//         if exist
-			//             Send local cache
-			//         else
-			//             Delete local /node.png*
-			//             Fetch and convert to /node.png-etag-<etagValue>
-			//             Send local cache
-			// else status code is 404
-			//      Delete /node.png*
-			//      Send 404
-			fmt.Println("Remote Addr is " + realRemoteAddr + ", fetching..")
-			statusCode, etagValue := GetRemoteImageInfo(realRemoteAddr)
-			if statusCode == 200 {
-				// Check local path: /node.png-etag-<etagValue>
-				localEtagImagePath := ExhaustPath + reqURI + "-etag-" + etagValue
-				if ImageExists(localEtagImagePath) {
-					c.SendFile(localEtagImagePath)
-				} else {
-					// Temporary store of remote file.
-					// ./remote-raw/node.png
-					CleanProxyCache(ExhaustPath + reqURI + "*")
-					localRemoteTmpPath := "./remote-raw" + reqURI
-					FetchRemoteImage(localRemoteTmpPath, realRemoteAddr)
-					q, _ := strconv.ParseFloat(QUALITY, 32)
-					_ = os.MkdirAll(path.Dir(localEtagImagePath), 0755)
-					err := WebpEncoder(localRemoteTmpPath, localEtagImagePath, float32(q), true, nil)
-					if err != nil {
-						fmt.Println(err)
-					}
-					c.SendFile(localEtagImagePath)
-				}
-			} else {
-				msg := fmt.Sprintf("Remote returned %d status code!", statusCode)
-				c.Send(msg)
-				log.Warn(msg)
-				c.SendStatus(statusCode)
-				CleanProxyCache(ExhaustPath + reqURI + "*")
-				return
-			}
-			// End Proxy Mode
 		} else {
-			// Check the original image for existence,
+			allowed = false
-			if !ImageExists(RawImageAbs) {
+		}
-				msg := "Image not found!"
+	}
-				c.Send(msg)
+	if !allowed {
-				log.Warn(msg)
+		msg := "File extension not allowed! " + imgFilename
-				c.SendStatus(404)
+		log.Warn(msg)
-				return
+		_ = c.Send([]byte(msg))
-			}
+		if imageExists(rawImageAbs) {
-
+			etag := genEtag(rawImageAbs)
-			_, WebpAbsPath := GenWebpAbs(RawImageAbs, ExhaustPath, ImgFilename, reqURI)
-
-			if ImageExists(WebpAbsPath) {
-				finalFile = WebpAbsPath
-			} else {
-				// we don't have abc.jpg.png1582558990.webp
-				// delete the old pic and convert a new one.
-				// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
-				destHalfFile := path.Clean(path.Join(WebpAbsPath, path.Dir(reqURI), ImgFilename))
-				matches, err := filepath.Glob(destHalfFile + "*")
-				if err != nil {
-					log.Error(err.Error())
-				} else {
-					// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558100.webp <- older ones will be removed
-					// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp <- keep the latest one
-					for _, p := range matches {
-						if strings.Compare(destHalfFile, p) != 0 {
-							_ = os.Remove(p)
-						}
-					}
-				}
-
-				//for webp, we need to create dir first
-				_ = os.MkdirAll(path.Dir(WebpAbsPath), 0755)
-				q, _ := strconv.ParseFloat(QUALITY, 32)
-				err = WebpEncoder(RawImageAbs, WebpAbsPath, float32(q), true, nil)
-
-				if err != nil {
-					log.Error(err)
-					c.SendStatus(400)
-					c.Send("Bad file!")
-					return
-				}
-				finalFile = WebpAbsPath
-			}
-			etag := GenEtag(finalFile)
 			c.Set("ETag", etag)
-			c.SendFile(finalFile)
+			return c.SendFile(rawImageAbs)
+		}
+		return errors.New(msg)
+	}
+
+	// Start Proxy Mode
+	if proxyMode {
+		// https://test.webp.sh/node.png
+		realRemoteAddr := config.ImgPath + reqURI
+		// Ping Remote for status code and etag info
+		fmt.Println("Remote Addr is " + realRemoteAddr + ", fetching..")
+		statusCode, etagValue := getRemoteImageInfo(realRemoteAddr)
+		if statusCode == 200 {
+			// Check local path: /node.png-etag-<etagValue>
+			localEtagImagePath := config.ExhaustPath + reqURI + "-etag-" + etagValue
+			if imageExists(localEtagImagePath) {
+				return c.SendFile(localEtagImagePath)
+			} else {
+				// Temporary store of remote file.
+				// ./remote-raw/node.png
+				cleanProxyCache(config.ExhaustPath + reqURI + "*")
+				localRemoteTmpPath := "./remote-raw" + reqURI
+				_ = fetchRemoteImage(localRemoteTmpPath, realRemoteAddr)
+				q, _ := strconv.ParseFloat(config.Quality, 32)
+				_ = os.MkdirAll(path.Dir(localEtagImagePath), 0755)
+				err := webpEncoder(localRemoteTmpPath, localEtagImagePath, float32(q), true, nil)
+				if err != nil {
+					fmt.Println(err)
+				}
+				return c.SendFile(localEtagImagePath)
+			}
+		} else {
+			msg := fmt.Sprintf("Remote returned %d status code!", statusCode)
+			_ = c.Send([]byte(msg))
+			log.Warn(msg)
+			_ = c.SendStatus(statusCode)
+			cleanProxyCache(config.ExhaustPath + reqURI + "*")
+			return errors.New(msg)
+		}
+		// End Proxy Mode
+	} else {
+		// Check the original image for existence,
+		if !imageExists(rawImageAbs) {
+			msg := "image not found"
+			_ = c.Send([]byte(msg))
+			log.Warn(msg)
+			_ = c.SendStatus(404)
+			return errors.New(msg)
 		}
 
+		_, webpAbsPath := genWebpAbs(rawImageAbs, config.ExhaustPath, imgFilename, reqURI)
+
+		if imageExists(webpAbsPath) {
+			finalFile = webpAbsPath
+		} else {
+			// we don't have abc.jpg.png1582558990.webp
+			// delete the old pic and convert a new one.
+			// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
+			destHalfFile := path.Clean(path.Join(webpAbsPath, path.Dir(reqURI), imgFilename))
+			matches, err := filepath.Glob(destHalfFile + "*")
+			if err != nil {
+				log.Error(err.Error())
+			} else {
+				// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558100.webp <- older ones will be removed
+				// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp <- keep the latest one
+				for _, p := range matches {
+					if strings.Compare(destHalfFile, p) != 0 {
+						_ = os.Remove(p)
+					}
+				}
+			}
+
+			//for webp, we need to create dir first
+			err = os.MkdirAll(path.Dir(webpAbsPath), 0755)
+			q, _ := strconv.ParseFloat(config.Quality, 32)
+			err = webpEncoder(rawImageAbs, webpAbsPath, float32(q), true, nil)
+
+			if err != nil {
+				log.Error(err)
+				_ = c.SendStatus(400)
+				_ = c.Send([]byte("Bad file!"))
+				return err
+			}
+			finalFile = webpAbsPath
+		}
+		etag := genEtag(finalFile)
+		c.Set("ETag", etag)
+		return c.SendFile(finalFile)
 	}
 }

router_test.go

@@ -0,0 +1,125 @@
+// webp_server_go - webp-server_test
+// 2020-11-09 11:55
+// Benny <benny.think@gmail.com>
+
+package main
+
+import (
+	"github.com/gofiber/fiber/v2"
+	"github.com/stretchr/testify/assert"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+var (
+	chromeUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36"
+	SafariUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1 Safari/605.1.15"
+)
+
+func TestConvert(t *testing.T) {
+	setupParam()
+	var testChromeLink = map[string]string{
+		"http://127.0.0.1:3333/webp_server.jpg": "image/webp",
+		"http://127.0.0.1:3333/webp_server.bmp": "image/webp",
+		"http://127.0.0.1:3333/webp_server.png": "image/webp",
+		"http://127.0.0.1:3333/empty.jpg":       "text/plain; charset=utf-8",
+		"http://127.0.0.1:3333/png.jpg":         "image/webp",
+		"http://127.0.0.1:3333/12314.jpg":       "text/plain; charset=utf-8",
+		"http://127.0.0.1:3333/dir1/inside.jpg": "image/webp",
+	}
+
+	var testSafariLink = map[string]string{
+		"http://127.0.0.1:3333/webp_server.jpg": "image/jpeg",
+		"http://127.0.0.1:3333/webp_server.bmp": "image/bmp",
+		"http://127.0.0.1:3333/webp_server.png": "image/png",
+		"http://127.0.0.1:3333/empty.jpg":       "text/plain; charset=utf-8",
+		"http://127.0.0.1:3333/png.jpg":         "image/png",
+		"http://127.0.0.1:3333/12314.jpg":       "text/plain; charset=utf-8",
+		"http://127.0.0.1:3333/dir1/inside.jpg": "image/jpeg",
+	}
+
+	var app = fiber.New()
+	app.Get("/*", convert)
+
+	// test Chrome
+	for url, respType := range testChromeLink {
+		_, data := requestToServer(url, app, chromeUA)
+		contentType := getFileContentType(data)
+		assert.Equal(t, respType, contentType)
+	}
+
+	// test Safari
+	for url, respType := range testSafariLink {
+		_, data := requestToServer(url, app, SafariUA)
+		contentType := getFileContentType(data)
+		assert.Equal(t, respType, contentType)
+	}
+
+}
+
+func TestConvertNotAllowed(t *testing.T) {
+	setupParam()
+	config.AllowedTypes = []string{"jpg", "png", "jpeg"}
+
+	var app = fiber.New()
+	app.Get("/*", convert)
+
+	// not allowed, but we have the file
+	url := "http://127.0.0.1:3333/webp_server.bmp"
+	_, data := requestToServer(url, app, chromeUA)
+	contentType := getFileContentType(data)
+	assert.Equal(t, "image/bmp", contentType)
+
+	// not allowed, random file
+	url = url + "hagdgd"
+	_, data = requestToServer(url, app, chromeUA)
+	assert.Contains(t, string(data), "File extension not allowed")
+
+}
+
+func TestConvertProxyModeBad(t *testing.T) {
+	setupParam()
+	proxyMode = true
+
+	var app = fiber.New()
+	app.Get("/*", convert)
+
+	// this is local image, should be 500
+	url := "http://127.0.0.1:3333/webp_server.bmp"
+	resp, _ := requestToServer(url, app, chromeUA)
+	assert.Equal(t, http.StatusInternalServerError, resp.StatusCode)
+
+}
+
+func TestConvertProxyModeWork(t *testing.T) {
+	setupParam()
+	proxyMode = true
+
+	var app = fiber.New()
+	app.Get("/*", convert)
+
+	config.ImgPath = "https://webp.sh"
+	url := "https://webp.sh/images/cover.jpg"
+
+	resp, data := requestToServer(url, app, chromeUA)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.Equal(t, "image/webp", getFileContentType(data))
+
+}
+
+func setupParam() {
+	// setup parameters here...
+	config.ImgPath = "./pics"
+	config.ExhaustPath = "./exhaust"
+	config.AllowedTypes = []string{"jpg", "png", "jpeg", "bmp"}
+}
+
+func requestToServer(url string, app *fiber.App, ua string) (*http.Response, []byte) {
+	req := httptest.NewRequest("GET", url, nil)
+	req.Header.Set("User-Agent", ua)
+	resp, _ := app.Test(req)
+	data, _ := ioutil.ReadAll(resp.Body)
+	return resp, data
+}

scripts/unit_test.sh

@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-
-# bash scripts/unit_test.sh
-# check $? for success or failure
-go test -v -cover encoder_test.go encoder.go helper.go
-go test -v -cover helper_test.go helper.go
-
-# if [[ $? -ne 0 ]] ; then
-#  echo "TEST FAILED!!! PLEASE DOUBLE CHECK."
-# fi

scripts/unix.sh

@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-cd ..
-git pull
-platform=$(uname -a)
-
-if [[ $platform =~ "Darwin" ]]
-then
-    go build -o webp-server-darwin-amd64 webp-server.go
-elif [[ $platform =~ "x86_64" ]];then
-    go build -o webp-server-unix-amd64 webp-server.go
-else
-    go build -o webp-server-linux-amd64 webp-server.go
-fi

scripts/webp_server.spec

@@ -1,5 +1,5 @@
 Name:    webp-server
-Version: 0.1.2
+Version: 0.2.1
 Release: 1%{?dist}
 Summary: Go version of WebP Server. A tool that will serve your JPG/PNGs as WebP format with compression, on-the-fly.
 

scripts/windows.bat

@@ -1,14 +0,0 @@
-cd ..
-git pull
-
-IF EXIST "%PROGRAMFILES(X86)%" (GOTO 64BIT) ELSE (GOTO 32BIT)
-:64BIT
-go build -o webp-server-windows-amd64.exe webp-server.go
-GOTO END
-
-:32BIT
-echo 32-bit...
-go build -o webp-server-windows-i386.exe webp-server.go
-GOTO END
-
-pause

update.go

@@ -50,7 +50,6 @@ func autoUpdate() {
 	}
 	data, _ := ioutil.ReadAll(resp.Body)
 	_ = os.Mkdir("update", 0755)
-	// TODO: checksum
 	err := ioutil.WriteFile(path.Join("update", filename), data, 0755)
 
 	if err == nil {

update_test.go

@@ -0,0 +1,27 @@
+// webp_server_go - update_test
+// 2020-11-10 09:36
+// Benny <benny.think@gmail.com>
+
+package main
+
+import (
+	"github.com/stretchr/testify/assert"
+	"os"
+	"testing"
+)
+
+func TestNormalAutoUpdate(t *testing.T) {
+	version = "0.0.1"
+	dir := "./update"
+	autoUpdate()
+	assert.NotEqual(t, 0, fileCount(dir))
+	_ = os.RemoveAll(dir)
+}
+
+func TestNoNeedAutoUpdate(t *testing.T) {
+	version = "99.99"
+	dir := "./update"
+	autoUpdate()
+	info, _ := os.Stat(dir)
+	assert.Nil(t, info)
+}

webp-server.go

@@ -5,42 +5,44 @@ import (
 	"flag"
 	"fmt"
 	"os"
-	"path"
 	"regexp"
 	"runtime"
 
-	"github.com/gofiber/fiber"
+	"github.com/gofiber/fiber/v2"
 	log "github.com/sirupsen/logrus"
 )
 
 type Config struct {
-	HOST         string
+	Host         string   `json:"HOST"`
-	PORT         string
+	Port         string   `json:"PORT"`
-	ImgPath      string `json:"IMG_PATH"`
+	ImgPath      string   `json:"IMG_PATH"`
-	QUALITY      string
+	Quality      string   `json:"QUALITY"`
 	AllowedTypes []string `json:"ALLOWED_TYPES"`
 	ExhaustPath  string   `json:"EXHAUST_PATH"`
 }
 
-const version = "0.2.0"
+var (
+	configPath                                                  string
+	jobs                                                        int
+	dumpConfig, dumpSystemd, verboseMode, prefetch, showVersion bool
 
-var configPath string
+	proxyMode bool
-var prefetch bool
+	config    Config
-var jobs int
+	version   = "0.2.1"
-var dumpConfig bool
+)
-var dumpSystemd bool
-var verboseMode bool
 
-const sampleConfig = `
+const (
+	sampleConfig = `
 {
-	"HOST": "127.0.0.1",
+  "HOST": "127.0.0.1",
-	"PORT": "3333",
+  "PORT": "3333",
-	"QUALITY": "80",
+  "QUALITY": "80",
-	"IMG_PATH": "/path/to/pics",
+  "IMG_PATH": "./pics",
-	"EXHAUST_PATH": "",
+  "EXHAUST_PATH": "./exhaust",
-	"ALLOWED_TYPES": ["jpg","png","jpeg","bmp"]
+  "ALLOWED_TYPES": ["jpg","png","jpeg","bmp"]
 }`
-const sampleSystemd = `
+
+	sampleSystemd = `
 [Unit]
 Description=WebP Server Go
 Documentation=https://github.com/webp-sh/webp_server_go
@@ -56,26 +58,27 @@ RestartSec=3s
 
 [Install]
 WantedBy=multi-user.target`
+)
 
 func loadConfig(path string) Config {
-	var config Config
 	jsonObject, err := os.Open(path)
 	if err != nil {
 		log.Fatal(err)
 	}
-	defer jsonObject.Close()
 	decoder := json.NewDecoder(jsonObject)
 	_ = decoder.Decode(&config)
+	_ = jsonObject.Close()
 	return config
 }
 
-func init() {
+func deferInit() {
 	flag.StringVar(&configPath, "config", "config.json", "/path/to/config.json. (Default: ./config.json)")
 	flag.BoolVar(&prefetch, "prefetch", false, "Prefetch and convert image to webp")
 	flag.IntVar(&jobs, "jobs", runtime.NumCPU(), "Prefetch thread, default is all.")
 	flag.BoolVar(&dumpConfig, "dump-config", false, "Print sample config.json")
 	flag.BoolVar(&dumpSystemd, "dump-systemd", false, "Print sample systemd service file.")
 	flag.BoolVar(&verboseMode, "v", false, "Verbose, print out debug info.")
+	flag.BoolVar(&showVersion, "V", false, "Show version information.")
 	flag.Parse()
 	// Logrus
 	log.SetOutput(os.Stdout)
@@ -99,6 +102,17 @@ func init() {
 }
 
 func main() {
+	// Our banner
+	banner := fmt.Sprintf(`
+▌ ▌   ▌  ▛▀▖ ▞▀▖                ▞▀▖
+▌▖▌▞▀▖▛▀▖▙▄▘ ▚▄ ▞▀▖▙▀▖▌ ▌▞▀▖▙▀▖ ▌▄▖▞▀▖
+▙▚▌▛▀ ▌ ▌▌   ▖ ▌▛▀ ▌  ▐▐ ▛▀ ▌   ▌ ▌▌ ▌
+▘ ▘▝▀▘▀▀ ▘   ▝▀ ▝▀▘▘   ▘ ▝▀▘▘   ▝▀ ▝▀
+
+Webp Server Go - v%s
+Develop by WebP Server team. https://github.com/webp-sh`, version)
+
+	deferInit()
 	// process cli params
 	if dumpConfig {
 		fmt.Println(sampleConfig)
@@ -108,49 +122,40 @@ func main() {
 		fmt.Println(sampleSystemd)
 		os.Exit(0)
 	}
+	if showVersion {
+		fmt.Printf("\n %c[1;32m%s%c[0m\n\n", 0x1B, banner+"", 0x1B)
+		os.Exit(0)
+	}
 
 	go autoUpdate()
-	config := loadConfig(configPath)
+	config = loadConfig(configPath)
 
-	HOST := config.HOST
-	PORT := config.PORT
 	// Check for remote address
 	matched, _ := regexp.MatchString(`^https?://`, config.ImgPath)
-	proxyMode := false
+	proxyMode = false
-	confImgPath := ""
 	if matched {
 		proxyMode = true
-		confImgPath = config.ImgPath
 	} else {
 		_, err := os.Stat(config.ImgPath)
 		if err != nil {
 			log.Fatalf("Your image path %s is incorrect.Please check and confirm.", config.ImgPath)
 		}
-		confImgPath = path.Clean(config.ImgPath)
-	}
-	QUALITY := config.QUALITY
-	AllowedTypes := config.AllowedTypes
-	var ExhaustPath string
-	if len(config.ExhaustPath) == 0 {
-		ExhaustPath = "./exhaust"
-	} else {
-		ExhaustPath = config.ExhaustPath
 	}
 
 	if prefetch {
-		go PrefetchImages(confImgPath, ExhaustPath, QUALITY)
+		go prefetchImages(config.ImgPath, config.ExhaustPath, config.Quality)
 	}
 
-	app := fiber.New()
+	app := fiber.New(fiber.Config{
-	app.Banner = false
+		ServerHeader:          "Webp-Server-Go",
-	app.Server = "WebP Server Go"
+		DisableStartupMessage: true,
+	})
+	listenAddress := config.Host + ":" + config.Port
+	app.Get("/*", convert)
 
-	ListenAddress := HOST + ":" + PORT
+	fmt.Printf("\n %c[1;32m%s%c[0m\n\n", 0x1B, banner, 0x1B)
+	fmt.Println("Webp-Server-Go is Running on http://" + listenAddress)
 
-	// Server Info
+	_ = app.Listen(listenAddress)
-	log.Infof("WebP Server %s %s", version, ListenAddress)
-
-	app.Get("/*", Convert(confImgPath, ExhaustPath, AllowedTypes, QUALITY, proxyMode))
-	app.Listen(ListenAddress)
 
 }

webp-server_test.go

@@ -0,0 +1,50 @@
+// webp_server_go - webp-server_test
+// 2020-11-10 09:41
+// Benny <benny.think@gmail.com>
+
+package main
+
+import (
+	"github.com/stretchr/testify/assert"
+	"net"
+	"runtime"
+	"testing"
+	"time"
+)
+
+// due to test limit, we can't test for cli param part.
+
+func TestLoadConfig(t *testing.T) {
+	c := loadConfig("./config.json")
+	assert.Equal(t, "./exhaust", c.ExhaustPath)
+	assert.Equal(t, "127.0.0.1", c.Host)
+	assert.Equal(t, "3333", c.Port)
+	assert.Equal(t, "80", c.Quality)
+	assert.Equal(t, "./pics", c.ImgPath)
+	assert.Equal(t, []string{"jpg", "png", "jpeg", "bmp"}, c.AllowedTypes)
+}
+
+func TestDeferInit(t *testing.T) {
+	// test initial value
+	assert.Equal(t, "", configPath)
+	assert.False(t, prefetch)
+	assert.Equal(t, false, dumpSystemd)
+	assert.Equal(t, false, dumpConfig)
+	assert.False(t, verboseMode)
+}
+
+func TestMainFunction(t *testing.T) {
+	go main()
+	time.Sleep(time.Second * 2)
+	// test read config value
+	assert.Equal(t, "config.json", configPath)
+	assert.False(t, prefetch)
+	assert.Equal(t, runtime.NumCPU(), jobs)
+	assert.Equal(t, false, dumpSystemd)
+	assert.Equal(t, false, dumpConfig)
+	assert.False(t, verboseMode)
+	// test port
+	conn, err := net.DialTimeout("tcp", net.JoinHostPort("127.0.0.1", "3333"), time.Second*2)
+	assert.Nil(t, err)
+	assert.NotNil(t, conn)
+}