From 08c333f3cd63f09bfacd0faf1880acd7da7c7631 Mon Sep 17 00:00:00 2001
From: Benny
Date: Sat, 21 Nov 2020 13:26:03 +0800
Subject: [PATCH] Fiber v2, code format, typo,test case, go 1.15 and more (#54)
* upgrade to fiber v2
* code format
* remove redundant variables
* remove useless exportable variables/functions
* go mod replace
use our own mirror now.
* add test case for converter, use deferInit to make test more simple
* remove useless file and fix typo
* Makefile change
* upgrade to go 1.15
* remove wrong go test comments
* complete test case, coverage, coverage badge
* Fix version typo
* config struct fix
* add banner, show version, add server header, remove fiber startup message
Co-authored-by: n0vad3v
---
.travis.yml | 10 +-
Makefile | 17 ++-
README.md | 2 +
config.json | 9 +-
encoder.go | 14 +-
encoder_test.go | 26 +++-
go.mod | 9 +-
helper.go | 22 +--
helper_test.go | 71 +++++++--
prefetch.go | 12 +-
prefetch_test.go | 29 ++++
router.go | 262 +++++++++++++++-----------------
router_test.go | 125 +++++++++++++++
scripts/unit_test.sh | 10 --
scripts/unix.sh | 13 --
scripts/webp_server.spec | 2 +-
{init => scripts}/webps.service | 0
scripts/windows.bat | 14 --
update.go | 1 -
update_test.go | 27 ++++
webp-server.go | 103 +++++++------
webp-server_test.go | 50 ++++++
22 files changed, 552 insertions(+), 276 deletions(-)
create mode 100644 prefetch_test.go
create mode 100644 router_test.go
delete mode 100644 scripts/unit_test.sh
delete mode 100644 scripts/unix.sh
rename {init => scripts}/webps.service (100%)
delete mode 100644 scripts/windows.bat
create mode 100644 update_test.go
create mode 100644 webp-server_test.go
diff --git a/.travis.yml b/.travis.yml
index 0ab5047..86a8742 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,7 +1,7 @@
language: go
-go:
- - 1.14.7
+go:
+ - 1.15.4
env: GO111MODULE=on
arch:
@@ -22,8 +22,7 @@ jobs:
os:
- linux
script:
- - go test -v -cover encoder_test.go encoder.go helper.go
- - go test -v -cover helper_test.go helper.go
+ - make test
deploy:
provider: releases
@@ -35,3 +34,6 @@ deploy:
repo: webp-sh/webp_server_go
tags: true
branch: master
+
+after_success:
+ - bash <(curl -s https://codecov.io/bash)
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 23eaaa8..3d5f6d0 100644
--- a/Makefile
+++ b/Makefile
@@ -10,6 +10,17 @@ else
ARCH=amd64
endif
-all: build
-build:
- ./scripts/build.sh $(OS) $(ARCH)
\ No newline at end of file
+default:
+ make clean
+ go build -o builds/webp-server-$(OS)-$(ARCH) .
+ ls builds
+all:
+ make clean
+ ./scripts/build.sh $(OS) $(ARCH)
+
+test:
+ go test -coverprofile=coverage.txt -covermode=atomic
+
+clean:
+ rm -rf builds
+ rm -rf prefetch
diff --git a/README.md b/README.md
index 4dc45e7..4fb7d24 100644
--- a/README.md
+++ b/README.md
@@ -3,6 +3,8 @@
+[](https://codecov.io/gh/webp-sh/webp_server_go)
+
[Documentation](https://docs.webp.sh/) | [Website](https://webp.sh/)
This is a Server based on Golang, which allows you to serve WebP images on the fly.
diff --git a/config.json b/config.json
index 743e590..b3385c1 100644
--- a/config.json
+++ b/config.json
@@ -3,6 +3,11 @@
"PORT": "3333",
"QUALITY": "80",
"IMG_PATH": "./pics",
- "EXHAUST_PATH": "",
- "ALLOWED_TYPES": ["jpg","png","jpeg","bmp"]
+ "EXHAUST_PATH": "./exhaust",
+ "ALLOWED_TYPES": [
+ "jpg",
+ "png",
+ "jpeg",
+ "bmp"
+ ]
}
\ No newline at end of file
diff --git a/encoder.go b/encoder.go
index 5528527..bf50b95 100644
--- a/encoder.go
+++ b/encoder.go
@@ -17,7 +17,7 @@ import (
"golang.org/x/image/bmp"
)
-func WebpEncoder(p1, p2 string, quality float32, Log bool, c chan int) (err error) {
+func webpEncoder(p1, p2 string, quality float32, Log bool, c chan int) (err error) {
// if convert fails, return error; success nil
log.Debugf("target: %s with quality of %f", path.Base(p1), quality)
@@ -26,11 +26,11 @@ func WebpEncoder(p1, p2 string, quality float32, Log bool, c chan int) (err erro
data, err := ioutil.ReadFile(p1)
if err != nil {
- ChanErr(c)
+ chanErr(c)
return
}
- contentType := GetFileContentType(data[:512])
+ contentType := getFileContentType(data[:512])
if strings.Contains(contentType, "jpeg") {
img, _ = jpeg.Decode(bytes.NewReader(data))
} else if strings.Contains(contentType, "png") {
@@ -47,18 +47,18 @@ func WebpEncoder(p1, p2 string, quality float32, Log bool, c chan int) (err erro
msg := "image file " + path.Base(p1) + " is corrupted or not supported"
log.Debug(msg)
err = errors.New(msg)
- ChanErr(c)
+ chanErr(c)
return
}
if err = webp.Encode(&buf, img, &webp.Options{Lossless: false, Quality: quality}); err != nil {
log.Error(err)
- ChanErr(c)
+ chanErr(c)
return
}
if err = ioutil.WriteFile(p2, buf.Bytes(), 0644); err != nil {
log.Error(err)
- ChanErr(c)
+ chanErr(c)
return
}
@@ -66,7 +66,7 @@ func WebpEncoder(p1, p2 string, quality float32, Log bool, c chan int) (err erro
log.Info("Save to " + p2 + " ok!\n")
}
- ChanErr(c)
+ chanErr(c)
return nil
}
diff --git a/encoder_test.go b/encoder_test.go
index 848e5f3..de752ba 100644
--- a/encoder_test.go
+++ b/encoder_test.go
@@ -1,15 +1,15 @@
package main
import (
+ "github.com/stretchr/testify/assert"
"io/ioutil"
"os"
"path/filepath"
"testing"
)
-//go test -v -cover encoder_test.go encoder.go helper.go
+//go test -v -cover .
func TestWebpEncoder(t *testing.T) {
-
var webp = "/tmp/test-result.webp"
var target = walker()
@@ -19,8 +19,24 @@ func TestWebpEncoder(t *testing.T) {
}
_ = os.Remove(webp)
+ // test error
+ err := webpEncoder("./pics/empty.jpg", webp, 80, true, nil)
+ assert.NotNil(t, err)
}
+func TestNonImage(t *testing.T) {
+ var webp = "/tmp/test-result.webp"
+ // test error
+ var err = webpEncoder("./pics/empty.jpg", webp, 80, true, nil)
+ assert.NotNil(t, err)
+}
+
+func TestWriteFail(t *testing.T) {
+ // test permission denied
+ var webp = "/123.webp"
+ var err = webpEncoder("./pics/png.jpg", webp, 80, true, nil)
+ assert.NotNil(t, err)
+}
func walker() []string {
var list []string
_ = filepath.Walk("./pics", func(path string, info os.FileInfo, err error) error {
@@ -34,8 +50,8 @@ func walker() []string {
func runEncoder(t *testing.T, file string, webp string) {
var c chan int
- //t.Logf("Convert from %s to %s", file, webp)
- var err = WebpEncoder(file, webp, 80, false, c)
+ //t.Logf("convert from %s to %s", file, webp)
+ var err = webpEncoder(file, webp, 80, true, c)
if file == "pics/empty.jpg" && err != nil {
t.Log("Empty file, that's okay.")
} else if err != nil {
@@ -43,7 +59,7 @@ func runEncoder(t *testing.T, file string, webp string) {
}
data, _ := ioutil.ReadFile(webp)
- types := GetFileContentType(data[:512])
+ types := getFileContentType(data[:512])
if types != "image/webp" {
t.Fatal("Fatal, file type is wrong!")
}
diff --git a/go.mod b/go.mod
index de70ada..4347c7b 100644
--- a/go.mod
+++ b/go.mod
@@ -1,11 +1,16 @@
module webp_server_go
-go 1.13
+go 1.15
require (
github.com/chai2010/webp v1.1.0
- github.com/gofiber/fiber v1.4.0
+ github.com/gofiber/fiber/v2 v2.1.4
github.com/sirupsen/logrus v1.6.0
github.com/stretchr/testify v1.3.0
golang.org/x/image v0.0.0-20200119044424-58c23975cae1
)
+
+replace (
+ github.com/gofiber/fiber/v2 v2.1.4 => github.com/webp-sh/fiber/v2 v2.1.4
+ github.com/chai2010/webp v1.1.0 => github.com/webp-sh/webp v1.1.1
+)
\ No newline at end of file
diff --git a/helper.go b/helper.go
index 8c806e3..ef65d90 100644
--- a/helper.go
+++ b/helper.go
@@ -15,20 +15,20 @@ import (
log "github.com/sirupsen/logrus"
)
-func ChanErr(ccc chan int) {
+func chanErr(ccc chan int) {
if ccc != nil {
ccc <- 1
}
}
-func GetFileContentType(buffer []byte) string {
+func getFileContentType(buffer []byte) string {
// Use the net/http package's handy DectectContentType function. Always returns a valid
// content-type by returning "application/octet-stream" if no others seemed to match.
contentType := http.DetectContentType(buffer)
return contentType
}
-func FileCount(dir string) int {
+func fileCount(dir string) int {
count := 0
_ = filepath.Walk(dir,
func(path string, info os.FileInfo, err error) error {
@@ -40,7 +40,7 @@ func FileCount(dir string) int {
return count
}
-func ImageExists(filename string) bool {
+func imageExists(filename string) bool {
info, err := os.Stat(filename)
if os.IsNotExist(err) {
return false
@@ -51,10 +51,11 @@ func ImageExists(filename string) bool {
// Check for remote filepath, e.g: https://test.webp.sh/node.png
// return StatusCode, etagValue
-func GetRemoteImageInfo(fileUrl string) (int, string) {
+func getRemoteImageInfo(fileUrl string) (int, string) {
res, err := http.Head(fileUrl)
if err != nil {
- log.Fatal("Connection to remote error!")
+ log.Errorln("Connection to remote error!")
+ return http.StatusInternalServerError, ""
}
if res.StatusCode != 404 {
etagValue := res.Header.Get("etag")
@@ -67,7 +68,7 @@ func GetRemoteImageInfo(fileUrl string) (int, string) {
return res.StatusCode, ""
}
-func FetchRemoteImage(filepath string, url string) error {
+func fetchRemoteImage(filepath string, url string) error {
resp, err := http.Get(url)
if err != nil {
return err
@@ -86,7 +87,7 @@ func FetchRemoteImage(filepath string, url string) error {
// Given /path/to/node.png
// Delete /path/to/node.png*
-func CleanProxyCache(cacheImagePath string) {
+func cleanProxyCache(cacheImagePath string) {
// Delete /node.png*
files, err := filepath.Glob(cacheImagePath + "*")
if err != nil {
@@ -99,11 +100,12 @@ func CleanProxyCache(cacheImagePath string) {
}
}
-func GenWebpAbs(RawImagePath string, ExhaustPath string, ImgFilename string, reqURI string) (string, string) {
+func genWebpAbs(RawImagePath string, ExhaustPath string, ImgFilename string, reqURI string) (string, string) {
// get file mod time
STAT, err := os.Stat(RawImagePath)
if err != nil {
log.Error(err.Error())
+ return "", ""
}
ModifiedTime := STAT.ModTime().Unix()
// webpFilename: abc.jpg.png -> abc.jpg.png1582558990.webp
@@ -116,7 +118,7 @@ func GenWebpAbs(RawImagePath string, ExhaustPath string, ImgFilename string, req
return cwd, WebpAbsolutePath
}
-func GenEtag(ImgAbsPath string) string {
+func genEtag(ImgAbsPath string) string {
data, err := ioutil.ReadFile(ImgAbsPath)
if err != nil {
log.Info(err)
diff --git a/helper_test.go b/helper_test.go
index a95133d..b272236 100644
--- a/helper_test.go
+++ b/helper_test.go
@@ -1,48 +1,49 @@
package main
import (
+ "io/ioutil"
+ "net/http"
+ "path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
-// test this file: go test -v -cover helper_test.go helper.go
-// test one function: go test -run TestGetFileContentType helper_test.go helper.go -v
+// test this file: go test -v -cover .
func TestGetFileContentType(t *testing.T) {
var data = []byte("hello")
var expected = "text/plain; charset=utf-8"
- var result = GetFileContentType(data)
+ var result = getFileContentType(data)
assert.Equalf(t, result, expected, "Result: [%s], Expected: [%s]", result, expected)
}
-// TODO: make a universal logging function
func TestFileCount(t *testing.T) {
var data = ".github"
var expected = 2
- var result = FileCount(data)
+ var result = fileCount(data)
assert.Equalf(t, result, expected, "Result: [%d], Expected: [%d]", result, expected)
}
func TestImageExists(t *testing.T) {
var data = "./pics/empty.jpg"
- var result = !ImageExists(data)
+ var result = !imageExists(data)
if result {
t.Errorf("Result: [%v], Expected: [%v]", result, false)
}
data = ".pics/empty2.jpg"
- result = ImageExists(data)
+ result = imageExists(data)
assert.Falsef(t, result, "Result: [%v], Expected: [%v]", result, false)
}
func TestGenWebpAbs(t *testing.T) {
- cwd, cooked := GenWebpAbs("./pics/webp_server.png", "/tmp",
+ cwd, cooked := genWebpAbs("./pics/webp_server.png", "/tmp",
"test", "a")
if !strings.Contains(cwd, "webp_server_go") {
t.Logf("Result: [%v], Expected: [%v]", cwd, "webp_server_go")
@@ -56,7 +57,7 @@ func TestGenWebpAbs(t *testing.T) {
func TestGenEtag(t *testing.T) {
var data = "./pics/png.jpg"
var expected = "W/\"1020764-262C0329\""
- var result = GenEtag(data)
+ var result = genEtag(data)
assert.Equalf(t, result, expected, "Result: [%s], Expected: [%s]", result, expected)
@@ -104,3 +105,55 @@ func TestGoOrigin(t *testing.T) {
}
}
+
+func TestChanErr(t *testing.T) {
+ var value = 2
+ var testC = make(chan int, 2)
+ testC <- value
+ chanErr(testC)
+ value = <-testC
+ assert.Equal(t, 2, value)
+}
+
+func TestGetRemoteImageInfo(t *testing.T) {
+ url := "http://github.com/favicon.ico"
+ statusCode, etag := getRemoteImageInfo(url)
+ assert.NotEqual(t, "", etag)
+ assert.Equal(t, statusCode, http.StatusOK)
+
+ // test non-exist url
+ url = "http://sdahjajda.com"
+ statusCode, etag = getRemoteImageInfo(url)
+ assert.Equal(t, "", etag)
+ assert.Equal(t, statusCode, http.StatusInternalServerError)
+}
+
+func TestFetchRemoteImage(t *testing.T) {
+ // test the normal one
+ fp := filepath.Join("./exhaust", "test.ico")
+ url := "http://github.com/favicon.ico"
+ err := fetchRemoteImage(fp, url)
+ assert.Equal(t, err, nil)
+ data, _ := ioutil.ReadFile(fp)
+ assert.Equal(t, "image/x-icon", getFileContentType(data))
+
+ // test can't create file
+ err = fetchRemoteImage("/", url)
+ assert.NotNil(t, err)
+
+ // test bad url
+ err = fetchRemoteImage(fp, "http://ahjdsgdsghja.cya")
+ assert.NotNil(t, err)
+}
+
+func TestCleanProxyCache(t *testing.T) {
+ // test normal situation
+ fp := filepath.Join("./exhaust", "sample.png.12345.webp")
+ _ = ioutil.WriteFile(fp, []byte("1234"), 0755)
+ assert.True(t, imageExists(fp))
+ cleanProxyCache(fp)
+ assert.False(t, imageExists(fp))
+
+ // test bad dir
+ cleanProxyCache("/aasdyg/dhj2/dagh")
+}
diff --git a/prefetch.go b/prefetch.go
index d257ad0..89e828f 100644
--- a/prefetch.go
+++ b/prefetch.go
@@ -4,15 +4,15 @@ import (
"fmt"
"os"
"path"
- "time"
"path/filepath"
"strconv"
"strings"
+ "time"
log "github.com/sirupsen/logrus"
)
-func PrefetchImages(confImgPath string, ExhaustPath string, QUALITY string) {
+func prefetchImages(confImgPath string, ExhaustPath string, QUALITY string) {
var sTime = time.Now()
// maximum ongoing prefetch is depending on your core of CPU
log.Infof("Prefetching using %d cores", jobs)
@@ -22,7 +22,7 @@ func PrefetchImages(confImgPath string, ExhaustPath string, QUALITY string) {
}
//prefetch, recursive through the dir
- all := FileCount(confImgPath)
+ all := fileCount(confImgPath)
count := 0
err := filepath.Walk(confImgPath,
func(picAbsPath string, info os.FileInfo, err error) error {
@@ -31,10 +31,10 @@ func PrefetchImages(confImgPath string, ExhaustPath string, QUALITY string) {
}
// RawImagePath string, ImgFilename string, reqURI string
proposedURI := strings.Replace(picAbsPath, confImgPath, "", 1)
- _, p2 := GenWebpAbs(picAbsPath, ExhaustPath, info.Name(), proposedURI)
+ _, p2 := genWebpAbs(picAbsPath, ExhaustPath, info.Name(), proposedURI)
q, _ := strconv.ParseFloat(QUALITY, 32)
_ = os.MkdirAll(path.Dir(p2), 0755)
- go WebpEncoder(picAbsPath, p2, float32(q), false, finishChan)
+ go webpEncoder(picAbsPath, p2, float32(q), false, finishChan)
count += <-finishChan
//progress bar
_, _ = fmt.Fprintf(os.Stdout, "[Webp Server started] - convert in progress: %d/%d\r", count, all)
@@ -45,6 +45,6 @@ func PrefetchImages(confImgPath string, ExhaustPath string, QUALITY string) {
}
elapsed := time.Since(sTime)
_, _ = fmt.Fprintf(os.Stdout, "Prefetch completeY(^_^)Y\n\n")
- _, _ = fmt.Fprintf(os.Stdout, "Convert %d file in %s (^_^)Y\n\n", count, elapsed)
+ _, _ = fmt.Fprintf(os.Stdout, "convert %d file in %s (^_^)Y\n\n", count, elapsed)
}
diff --git a/prefetch_test.go b/prefetch_test.go
new file mode 100644
index 0000000..24e3e67
--- /dev/null
+++ b/prefetch_test.go
@@ -0,0 +1,29 @@
+// webp_server_go - prefetch_test.go
+// 2020-11-10 09:27
+// Benny
+
+package main
+
+import (
+ "github.com/stretchr/testify/assert"
+ "os"
+ "testing"
+)
+
+func TestPrefetchImages(t *testing.T) {
+ // single thread
+ fp := "./prefetch"
+ _ = os.Mkdir(fp, 0755)
+ prefetchImages("./pics", "./prefetch", "80")
+ count := fileCount("./prefetch")
+ assert.Equal(t, 6, count)
+ _ = os.RemoveAll(fp)
+
+ // concurrency
+ jobs = 2
+ _ = os.Mkdir(fp, 0755)
+ prefetchImages("./pics", "./prefetch", "80")
+ count = fileCount("./prefetch")
+ assert.Equal(t, 4, count)
+ _ = os.RemoveAll(fp)
+}
diff --git a/router.go b/router.go
index 1400f0d..fbafff6 100644
--- a/router.go
+++ b/router.go
@@ -1,159 +1,141 @@
package main
import (
+ "errors"
"fmt"
+ "github.com/gofiber/fiber/v2"
+ log "github.com/sirupsen/logrus"
"os"
"path"
"path/filepath"
"strconv"
"strings"
-
- log "github.com/sirupsen/logrus"
-
- "github.com/gofiber/fiber"
)
-func Convert(ImgPath string, ExhaustPath string, AllowedTypes []string, QUALITY string, proxyMode bool) func(c *fiber.Ctx) {
- return func(c *fiber.Ctx) {
- //basic vars
- var reqURI = c.Path() // /mypic/123.jpg
- var RawImageAbs = path.Join(ImgPath, reqURI) // /home/xxx/mypic/123.jpg
- var ImgFilename = path.Base(reqURI) // pure filename, 123.jpg
- var finalFile string // We'll only need one c.sendFile()
- var UA = c.Get("User-Agent")
- done := goOrigin(UA)
- if done {
- log.Infof("A Safari/IE/whatever user has arrived...%s", UA)
- // Check for Safari users. If they're Safari, just simply ignore everything.
+func convert(c *fiber.Ctx) error {
+ //basic vars
+ var reqURI = c.Path() // /mypic/123.jpg
+ var rawImageAbs = path.Join(config.ImgPath, reqURI) // /home/xxx/mypic/123.jpg
+ var imgFilename = path.Base(reqURI) // pure filename, 123.jpg
+ var finalFile string // We'll only need one c.sendFile()
+ var UA = c.Get("User-Agent")
+ done := goOrigin(UA)
+ if done {
+ log.Infof("A Safari/IE/whatever user has arrived...%s", UA)
+ // Check for Safari users. If they're Safari, just simply ignore everything.
- etag := GenEtag(RawImageAbs)
- c.Set("ETag", etag)
- c.SendFile(RawImageAbs)
- return
- }
- log.Debugf("Incoming connection from %s@%s with %s", UA, c.IP(), ImgFilename)
+ etag := genEtag(rawImageAbs)
+ c.Set("ETag", etag)
+ return c.SendFile(rawImageAbs)
+ }
+ log.Debugf("Incoming connection from %s@%s with %s", UA, c.IP(), imgFilename)
- // check ext
- // TODO: may remove this function. Check in Nginx.
- var allowed = false
- for _, ext := range AllowedTypes {
- haystack := strings.ToLower(ImgFilename)
- needle := strings.ToLower("." + ext)
- if strings.HasSuffix(haystack, needle) {
- allowed = true
- break
- } else {
- allowed = false
- }
- }
- if !allowed {
- msg := "File extension not allowed! " + ImgFilename
- log.Warn(msg)
- c.Send(msg)
- if ImageExists(RawImageAbs) {
- etag := GenEtag(RawImageAbs)
- c.Set("ETag", etag)
- c.SendFile(RawImageAbs)
- }
- return
- }
-
- // Start Proxy Mode
- if proxyMode {
- // https://test.webp.sh/node.png
- realRemoteAddr := ImgPath + reqURI
- // Ping Remote for status code and etag info
-
- // If status code is 200
- // Check for local /node.png-etag-
- // if exist
- // Send local cache
- // else
- // Delete local /node.png*
- // Fetch and convert to /node.png-etag-
- // Send local cache
- // else status code is 404
- // Delete /node.png*
- // Send 404
- fmt.Println("Remote Addr is " + realRemoteAddr + ", fetching..")
- statusCode, etagValue := GetRemoteImageInfo(realRemoteAddr)
- if statusCode == 200 {
- // Check local path: /node.png-etag-
- localEtagImagePath := ExhaustPath + reqURI + "-etag-" + etagValue
- if ImageExists(localEtagImagePath) {
- c.SendFile(localEtagImagePath)
- } else {
- // Temporary store of remote file.
- // ./remote-raw/node.png
- CleanProxyCache(ExhaustPath + reqURI + "*")
- localRemoteTmpPath := "./remote-raw" + reqURI
- FetchRemoteImage(localRemoteTmpPath, realRemoteAddr)
- q, _ := strconv.ParseFloat(QUALITY, 32)
- _ = os.MkdirAll(path.Dir(localEtagImagePath), 0755)
- err := WebpEncoder(localRemoteTmpPath, localEtagImagePath, float32(q), true, nil)
- if err != nil {
- fmt.Println(err)
- }
- c.SendFile(localEtagImagePath)
- }
- } else {
- msg := fmt.Sprintf("Remote returned %d status code!", statusCode)
- c.Send(msg)
- log.Warn(msg)
- c.SendStatus(statusCode)
- CleanProxyCache(ExhaustPath + reqURI + "*")
- return
- }
- // End Proxy Mode
+ // check ext
+ var allowed = false
+ for _, ext := range config.AllowedTypes {
+ haystack := strings.ToLower(imgFilename)
+ needle := strings.ToLower("." + ext)
+ if strings.HasSuffix(haystack, needle) {
+ allowed = true
+ break
} else {
- // Check the original image for existence,
- if !ImageExists(RawImageAbs) {
- msg := "Image not found!"
- c.Send(msg)
- log.Warn(msg)
- c.SendStatus(404)
- return
- }
-
- _, WebpAbsPath := GenWebpAbs(RawImageAbs, ExhaustPath, ImgFilename, reqURI)
-
- if ImageExists(WebpAbsPath) {
- finalFile = WebpAbsPath
- } else {
- // we don't have abc.jpg.png1582558990.webp
- // delete the old pic and convert a new one.
- // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
- destHalfFile := path.Clean(path.Join(WebpAbsPath, path.Dir(reqURI), ImgFilename))
- matches, err := filepath.Glob(destHalfFile + "*")
- if err != nil {
- log.Error(err.Error())
- } else {
- // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558100.webp <- older ones will be removed
- // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp <- keep the latest one
- for _, p := range matches {
- if strings.Compare(destHalfFile, p) != 0 {
- _ = os.Remove(p)
- }
- }
- }
-
- //for webp, we need to create dir first
- _ = os.MkdirAll(path.Dir(WebpAbsPath), 0755)
- q, _ := strconv.ParseFloat(QUALITY, 32)
- err = WebpEncoder(RawImageAbs, WebpAbsPath, float32(q), true, nil)
-
- if err != nil {
- log.Error(err)
- c.SendStatus(400)
- c.Send("Bad file!")
- return
- }
- finalFile = WebpAbsPath
- }
- etag := GenEtag(finalFile)
+ allowed = false
+ }
+ }
+ if !allowed {
+ msg := "File extension not allowed! " + imgFilename
+ log.Warn(msg)
+ _ = c.Send([]byte(msg))
+ if imageExists(rawImageAbs) {
+ etag := genEtag(rawImageAbs)
c.Set("ETag", etag)
- c.SendFile(finalFile)
+ return c.SendFile(rawImageAbs)
+ }
+ return errors.New(msg)
+ }
+
+ // Start Proxy Mode
+ if proxyMode {
+ // https://test.webp.sh/node.png
+ realRemoteAddr := config.ImgPath + reqURI
+ // Ping Remote for status code and etag info
+ fmt.Println("Remote Addr is " + realRemoteAddr + ", fetching..")
+ statusCode, etagValue := getRemoteImageInfo(realRemoteAddr)
+ if statusCode == 200 {
+ // Check local path: /node.png-etag-
+ localEtagImagePath := config.ExhaustPath + reqURI + "-etag-" + etagValue
+ if imageExists(localEtagImagePath) {
+ return c.SendFile(localEtagImagePath)
+ } else {
+ // Temporary store of remote file.
+ // ./remote-raw/node.png
+ cleanProxyCache(config.ExhaustPath + reqURI + "*")
+ localRemoteTmpPath := "./remote-raw" + reqURI
+ _ = fetchRemoteImage(localRemoteTmpPath, realRemoteAddr)
+ q, _ := strconv.ParseFloat(config.Quality, 32)
+ _ = os.MkdirAll(path.Dir(localEtagImagePath), 0755)
+ err := webpEncoder(localRemoteTmpPath, localEtagImagePath, float32(q), true, nil)
+ if err != nil {
+ fmt.Println(err)
+ }
+ return c.SendFile(localEtagImagePath)
+ }
+ } else {
+ msg := fmt.Sprintf("Remote returned %d status code!", statusCode)
+ _ = c.Send([]byte(msg))
+ log.Warn(msg)
+ _ = c.SendStatus(statusCode)
+ cleanProxyCache(config.ExhaustPath + reqURI + "*")
+ return errors.New(msg)
+ }
+ // End Proxy Mode
+ } else {
+ // Check the original image for existence,
+ if !imageExists(rawImageAbs) {
+ msg := "image not found"
+ _ = c.Send([]byte(msg))
+ log.Warn(msg)
+ _ = c.SendStatus(404)
+ return errors.New(msg)
}
+ _, webpAbsPath := genWebpAbs(rawImageAbs, config.ExhaustPath, imgFilename, reqURI)
+
+ if imageExists(webpAbsPath) {
+ finalFile = webpAbsPath
+ } else {
+ // we don't have abc.jpg.png1582558990.webp
+ // delete the old pic and convert a new one.
+ // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
+ destHalfFile := path.Clean(path.Join(webpAbsPath, path.Dir(reqURI), imgFilename))
+ matches, err := filepath.Glob(destHalfFile + "*")
+ if err != nil {
+ log.Error(err.Error())
+ } else {
+ // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558100.webp <- older ones will be removed
+ // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp <- keep the latest one
+ for _, p := range matches {
+ if strings.Compare(destHalfFile, p) != 0 {
+ _ = os.Remove(p)
+ }
+ }
+ }
+
+ //for webp, we need to create dir first
+ err = os.MkdirAll(path.Dir(webpAbsPath), 0755)
+ q, _ := strconv.ParseFloat(config.Quality, 32)
+ err = webpEncoder(rawImageAbs, webpAbsPath, float32(q), true, nil)
+
+ if err != nil {
+ log.Error(err)
+ _ = c.SendStatus(400)
+ _ = c.Send([]byte("Bad file!"))
+ return err
+ }
+ finalFile = webpAbsPath
+ }
+ etag := genEtag(finalFile)
+ c.Set("ETag", etag)
+ return c.SendFile(finalFile)
}
}
diff --git a/router_test.go b/router_test.go
new file mode 100644
index 0000000..fdfe3d1
--- /dev/null
+++ b/router_test.go
@@ -0,0 +1,125 @@
+// webp_server_go - webp-server_test
+// 2020-11-09 11:55
+// Benny
+
+package main
+
+import (
+ "github.com/gofiber/fiber/v2"
+ "github.com/stretchr/testify/assert"
+ "io/ioutil"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+)
+
+var (
+ chromeUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36"
+ SafariUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1 Safari/605.1.15"
+)
+
+func TestConvert(t *testing.T) {
+ setupParam()
+ var testChromeLink = map[string]string{
+ "http://127.0.0.1:3333/webp_server.jpg": "image/webp",
+ "http://127.0.0.1:3333/webp_server.bmp": "image/webp",
+ "http://127.0.0.1:3333/webp_server.png": "image/webp",
+ "http://127.0.0.1:3333/empty.jpg": "text/plain; charset=utf-8",
+ "http://127.0.0.1:3333/png.jpg": "image/webp",
+ "http://127.0.0.1:3333/12314.jpg": "text/plain; charset=utf-8",
+ "http://127.0.0.1:3333/dir1/inside.jpg": "image/webp",
+ }
+
+ var testSafariLink = map[string]string{
+ "http://127.0.0.1:3333/webp_server.jpg": "image/jpeg",
+ "http://127.0.0.1:3333/webp_server.bmp": "image/bmp",
+ "http://127.0.0.1:3333/webp_server.png": "image/png",
+ "http://127.0.0.1:3333/empty.jpg": "text/plain; charset=utf-8",
+ "http://127.0.0.1:3333/png.jpg": "image/png",
+ "http://127.0.0.1:3333/12314.jpg": "text/plain; charset=utf-8",
+ "http://127.0.0.1:3333/dir1/inside.jpg": "image/jpeg",
+ }
+
+ var app = fiber.New()
+ app.Get("/*", convert)
+
+ // test Chrome
+ for url, respType := range testChromeLink {
+ _, data := requestToServer(url, app, chromeUA)
+ contentType := getFileContentType(data)
+ assert.Equal(t, respType, contentType)
+ }
+
+ // test Safari
+ for url, respType := range testSafariLink {
+ _, data := requestToServer(url, app, SafariUA)
+ contentType := getFileContentType(data)
+ assert.Equal(t, respType, contentType)
+ }
+
+}
+
+func TestConvertNotAllowed(t *testing.T) {
+ setupParam()
+ config.AllowedTypes = []string{"jpg", "png", "jpeg"}
+
+ var app = fiber.New()
+ app.Get("/*", convert)
+
+ // not allowed, but we have the file
+ url := "http://127.0.0.1:3333/webp_server.bmp"
+ _, data := requestToServer(url, app, chromeUA)
+ contentType := getFileContentType(data)
+ assert.Equal(t, "image/bmp", contentType)
+
+ // not allowed, random file
+ url = url + "hagdgd"
+ _, data = requestToServer(url, app, chromeUA)
+ assert.Contains(t, string(data), "File extension not allowed")
+
+}
+
+func TestConvertProxyModeBad(t *testing.T) {
+ setupParam()
+ proxyMode = true
+
+ var app = fiber.New()
+ app.Get("/*", convert)
+
+ // this is local image, should be 500
+ url := "http://127.0.0.1:3333/webp_server.bmp"
+ resp, _ := requestToServer(url, app, chromeUA)
+ assert.Equal(t, http.StatusInternalServerError, resp.StatusCode)
+
+}
+
+func TestConvertProxyModeWork(t *testing.T) {
+ setupParam()
+ proxyMode = true
+
+ var app = fiber.New()
+ app.Get("/*", convert)
+
+ config.ImgPath = "https://webp.sh"
+ url := "https://webp.sh/images/cover.jpg"
+
+ resp, data := requestToServer(url, app, chromeUA)
+ assert.Equal(t, http.StatusOK, resp.StatusCode)
+ assert.Equal(t, "image/webp", getFileContentType(data))
+
+}
+
+func setupParam() {
+ // setup parameters here...
+ config.ImgPath = "./pics"
+ config.ExhaustPath = "./exhaust"
+ config.AllowedTypes = []string{"jpg", "png", "jpeg", "bmp"}
+}
+
+func requestToServer(url string, app *fiber.App, ua string) (*http.Response, []byte) {
+ req := httptest.NewRequest("GET", url, nil)
+ req.Header.Set("User-Agent", ua)
+ resp, _ := app.Test(req)
+ data, _ := ioutil.ReadAll(resp.Body)
+ return resp, data
+}
diff --git a/scripts/unit_test.sh b/scripts/unit_test.sh
deleted file mode 100644
index 2900710..0000000
--- a/scripts/unit_test.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-
-# bash scripts/unit_test.sh
-# check $? for success or failure
-go test -v -cover encoder_test.go encoder.go helper.go
-go test -v -cover helper_test.go helper.go
-
-# if [[ $? -ne 0 ]] ; then
-# echo "TEST FAILED!!! PLEASE DOUBLE CHECK."
-# fi
diff --git a/scripts/unix.sh b/scripts/unix.sh
deleted file mode 100644
index c915834..0000000
--- a/scripts/unix.sh
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-cd ..
-git pull
-platform=$(uname -a)
-
-if [[ $platform =~ "Darwin" ]]
-then
- go build -o webp-server-darwin-amd64 webp-server.go
-elif [[ $platform =~ "x86_64" ]];then
- go build -o webp-server-unix-amd64 webp-server.go
-else
- go build -o webp-server-linux-amd64 webp-server.go
-fi
diff --git a/scripts/webp_server.spec b/scripts/webp_server.spec
index b6e5613..9a8d61d 100644
--- a/scripts/webp_server.spec
+++ b/scripts/webp_server.spec
@@ -1,5 +1,5 @@
Name: webp-server
-Version: 0.1.2
+Version: 0.2.1
Release: 1%{?dist}
Summary: Go version of WebP Server. A tool that will serve your JPG/PNGs as WebP format with compression, on-the-fly.
diff --git a/init/webps.service b/scripts/webps.service
similarity index 100%
rename from init/webps.service
rename to scripts/webps.service
diff --git a/scripts/windows.bat b/scripts/windows.bat
deleted file mode 100644
index 6cc15b6..0000000
--- a/scripts/windows.bat
+++ /dev/null
@@ -1,14 +0,0 @@
-cd ..
-git pull
-
-IF EXIST "%PROGRAMFILES(X86)%" (GOTO 64BIT) ELSE (GOTO 32BIT)
-:64BIT
-go build -o webp-server-windows-amd64.exe webp-server.go
-GOTO END
-
-:32BIT
-echo 32-bit...
-go build -o webp-server-windows-i386.exe webp-server.go
-GOTO END
-
-pause
diff --git a/update.go b/update.go
index 033b292..e7ce14e 100644
--- a/update.go
+++ b/update.go
@@ -50,7 +50,6 @@ func autoUpdate() {
}
data, _ := ioutil.ReadAll(resp.Body)
_ = os.Mkdir("update", 0755)
- // TODO: checksum
err := ioutil.WriteFile(path.Join("update", filename), data, 0755)
if err == nil {
diff --git a/update_test.go b/update_test.go
new file mode 100644
index 0000000..e50cc12
--- /dev/null
+++ b/update_test.go
@@ -0,0 +1,27 @@
+// webp_server_go - update_test
+// 2020-11-10 09:36
+// Benny
+
+package main
+
+import (
+ "github.com/stretchr/testify/assert"
+ "os"
+ "testing"
+)
+
+func TestNormalAutoUpdate(t *testing.T) {
+ version = "0.0.1"
+ dir := "./update"
+ autoUpdate()
+ assert.NotEqual(t, 0, fileCount(dir))
+ _ = os.RemoveAll(dir)
+}
+
+func TestNoNeedAutoUpdate(t *testing.T) {
+ version = "99.99"
+ dir := "./update"
+ autoUpdate()
+ info, _ := os.Stat(dir)
+ assert.Nil(t, info)
+}
diff --git a/webp-server.go b/webp-server.go
index aa716ae..0f7c759 100644
--- a/webp-server.go
+++ b/webp-server.go
@@ -5,42 +5,44 @@ import (
"flag"
"fmt"
"os"
- "path"
"regexp"
"runtime"
- "github.com/gofiber/fiber"
+ "github.com/gofiber/fiber/v2"
log "github.com/sirupsen/logrus"
)
type Config struct {
- HOST string
- PORT string
- ImgPath string `json:"IMG_PATH"`
- QUALITY string
+ Host string `json:"HOST"`
+ Port string `json:"PORT"`
+ ImgPath string `json:"IMG_PATH"`
+ Quality string `json:"QUALITY"`
AllowedTypes []string `json:"ALLOWED_TYPES"`
ExhaustPath string `json:"EXHAUST_PATH"`
}
-const version = "0.2.0"
+var (
+ configPath string
+ jobs int
+ dumpConfig, dumpSystemd, verboseMode, prefetch, showVersion bool
-var configPath string
-var prefetch bool
-var jobs int
-var dumpConfig bool
-var dumpSystemd bool
-var verboseMode bool
+ proxyMode bool
+ config Config
+ version = "0.2.1"
+)
-const sampleConfig = `
+const (
+ sampleConfig = `
{
- "HOST": "127.0.0.1",
- "PORT": "3333",
- "QUALITY": "80",
- "IMG_PATH": "/path/to/pics",
- "EXHAUST_PATH": "",
- "ALLOWED_TYPES": ["jpg","png","jpeg","bmp"]
+ "HOST": "127.0.0.1",
+ "PORT": "3333",
+ "QUALITY": "80",
+ "IMG_PATH": "./pics",
+ "EXHAUST_PATH": "./exhaust",
+ "ALLOWED_TYPES": ["jpg","png","jpeg","bmp"]
}`
-const sampleSystemd = `
+
+ sampleSystemd = `
[Unit]
Description=WebP Server Go
Documentation=https://github.com/webp-sh/webp_server_go
@@ -56,26 +58,27 @@ RestartSec=3s
[Install]
WantedBy=multi-user.target`
+)
func loadConfig(path string) Config {
- var config Config
jsonObject, err := os.Open(path)
if err != nil {
log.Fatal(err)
}
- defer jsonObject.Close()
decoder := json.NewDecoder(jsonObject)
_ = decoder.Decode(&config)
+ _ = jsonObject.Close()
return config
}
-func init() {
+func deferInit() {
flag.StringVar(&configPath, "config", "config.json", "/path/to/config.json. (Default: ./config.json)")
flag.BoolVar(&prefetch, "prefetch", false, "Prefetch and convert image to webp")
flag.IntVar(&jobs, "jobs", runtime.NumCPU(), "Prefetch thread, default is all.")
flag.BoolVar(&dumpConfig, "dump-config", false, "Print sample config.json")
flag.BoolVar(&dumpSystemd, "dump-systemd", false, "Print sample systemd service file.")
flag.BoolVar(&verboseMode, "v", false, "Verbose, print out debug info.")
+ flag.BoolVar(&showVersion, "V", false, "Show version information.")
flag.Parse()
// Logrus
log.SetOutput(os.Stdout)
@@ -99,6 +102,17 @@ func init() {
}
func main() {
+ // Our banner
+ banner := fmt.Sprintf(`
+▌ ▌ ▌ ▛▀▖ ▞▀▖ ▞▀▖
+▌▖▌▞▀▖▛▀▖▙▄▘ ▚▄ ▞▀▖▙▀▖▌ ▌▞▀▖▙▀▖ ▌▄▖▞▀▖
+▙▚▌▛▀ ▌ ▌▌ ▖ ▌▛▀ ▌ ▐▐ ▛▀ ▌ ▌ ▌▌ ▌
+▘ ▘▝▀▘▀▀ ▘ ▝▀ ▝▀▘▘ ▘ ▝▀▘▘ ▝▀ ▝▀
+
+Webp Server Go - v%s
+Develop by WebP Server team. https://github.com/webp-sh`, version)
+
+ deferInit()
// process cli params
if dumpConfig {
fmt.Println(sampleConfig)
@@ -108,49 +122,40 @@ func main() {
fmt.Println(sampleSystemd)
os.Exit(0)
}
+ if showVersion {
+ fmt.Printf("\n %c[1;32m%s%c[0m\n\n", 0x1B, banner+"", 0x1B)
+ os.Exit(0)
+ }
go autoUpdate()
- config := loadConfig(configPath)
+ config = loadConfig(configPath)
- HOST := config.HOST
- PORT := config.PORT
// Check for remote address
matched, _ := regexp.MatchString(`^https?://`, config.ImgPath)
- proxyMode := false
- confImgPath := ""
+ proxyMode = false
if matched {
proxyMode = true
- confImgPath = config.ImgPath
} else {
_, err := os.Stat(config.ImgPath)
if err != nil {
log.Fatalf("Your image path %s is incorrect.Please check and confirm.", config.ImgPath)
}
- confImgPath = path.Clean(config.ImgPath)
- }
- QUALITY := config.QUALITY
- AllowedTypes := config.AllowedTypes
- var ExhaustPath string
- if len(config.ExhaustPath) == 0 {
- ExhaustPath = "./exhaust"
- } else {
- ExhaustPath = config.ExhaustPath
}
if prefetch {
- go PrefetchImages(confImgPath, ExhaustPath, QUALITY)
+ go prefetchImages(config.ImgPath, config.ExhaustPath, config.Quality)
}
- app := fiber.New()
- app.Banner = false
- app.Server = "WebP Server Go"
+ app := fiber.New(fiber.Config{
+ ServerHeader: "Webp-Server-Go",
+ DisableStartupMessage: true,
+ })
+ listenAddress := config.Host + ":" + config.Port
+ app.Get("/*", convert)
- ListenAddress := HOST + ":" + PORT
+ fmt.Printf("\n %c[1;32m%s%c[0m\n\n", 0x1B, banner, 0x1B)
+ fmt.Println("Webp-Server-Go is Running on http://" + listenAddress)
- // Server Info
- log.Infof("WebP Server %s %s", version, ListenAddress)
-
- app.Get("/*", Convert(confImgPath, ExhaustPath, AllowedTypes, QUALITY, proxyMode))
- app.Listen(ListenAddress)
+ _ = app.Listen(listenAddress)
}
diff --git a/webp-server_test.go b/webp-server_test.go
new file mode 100644
index 0000000..d4565df
--- /dev/null
+++ b/webp-server_test.go
@@ -0,0 +1,50 @@
+// webp_server_go - webp-server_test
+// 2020-11-10 09:41
+// Benny
+
+package main
+
+import (
+ "github.com/stretchr/testify/assert"
+ "net"
+ "runtime"
+ "testing"
+ "time"
+)
+
+// due to test limit, we can't test for cli param part.
+
+func TestLoadConfig(t *testing.T) {
+ c := loadConfig("./config.json")
+ assert.Equal(t, "./exhaust", c.ExhaustPath)
+ assert.Equal(t, "127.0.0.1", c.Host)
+ assert.Equal(t, "3333", c.Port)
+ assert.Equal(t, "80", c.Quality)
+ assert.Equal(t, "./pics", c.ImgPath)
+ assert.Equal(t, []string{"jpg", "png", "jpeg", "bmp"}, c.AllowedTypes)
+}
+
+func TestDeferInit(t *testing.T) {
+ // test initial value
+ assert.Equal(t, "", configPath)
+ assert.False(t, prefetch)
+ assert.Equal(t, false, dumpSystemd)
+ assert.Equal(t, false, dumpConfig)
+ assert.False(t, verboseMode)
+}
+
+func TestMainFunction(t *testing.T) {
+ go main()
+ time.Sleep(time.Second * 2)
+ // test read config value
+ assert.Equal(t, "config.json", configPath)
+ assert.False(t, prefetch)
+ assert.Equal(t, runtime.NumCPU(), jobs)
+ assert.Equal(t, false, dumpSystemd)
+ assert.Equal(t, false, dumpConfig)
+ assert.False(t, verboseMode)
+ // test port
+ conn, err := net.DialTimeout("tcp", net.JoinHostPort("127.0.0.1", "3333"), time.Second*2)
+ assert.Nil(t, err)
+ assert.NotNil(t, conn)
+}