diff --git a/.travis.yml b/.travis.yml index 0ab5047..86a8742 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: go -go: - - 1.14.7 +go: + - 1.15.4 env: GO111MODULE=on arch: @@ -22,8 +22,7 @@ jobs: os: - linux script: - - go test -v -cover encoder_test.go encoder.go helper.go - - go test -v -cover helper_test.go helper.go + - make test deploy: provider: releases @@ -35,3 +34,6 @@ deploy: repo: webp-sh/webp_server_go tags: true branch: master + +after_success: + - bash <(curl -s https://codecov.io/bash) \ No newline at end of file diff --git a/Makefile b/Makefile index 23eaaa8..3d5f6d0 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,17 @@ else ARCH=amd64 endif -all: build -build: - ./scripts/build.sh $(OS) $(ARCH) \ No newline at end of file +default: + make clean + go build -o builds/webp-server-$(OS)-$(ARCH) . + ls builds +all: + make clean + ./scripts/build.sh $(OS) $(ARCH) + +test: + go test -coverprofile=coverage.txt -covermode=atomic + +clean: + rm -rf builds + rm -rf prefetch diff --git a/README.md b/README.md index 4dc45e7..4fb7d24 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,8 @@

+[![codecov](https://codecov.io/gh/webp-sh/webp_server_go/branch/master/graph/badge.svg?token=VR3BMZME65)](https://codecov.io/gh/webp-sh/webp_server_go) + [Documentation](https://docs.webp.sh/) | [Website](https://webp.sh/) This is a Server based on Golang, which allows you to serve WebP images on the fly. diff --git a/config.json b/config.json index 743e590..b3385c1 100644 --- a/config.json +++ b/config.json @@ -3,6 +3,11 @@ "PORT": "3333", "QUALITY": "80", "IMG_PATH": "./pics", - "EXHAUST_PATH": "", - "ALLOWED_TYPES": ["jpg","png","jpeg","bmp"] + "EXHAUST_PATH": "./exhaust", + "ALLOWED_TYPES": [ + "jpg", + "png", + "jpeg", + "bmp" + ] } \ No newline at end of file diff --git a/encoder.go b/encoder.go index 5528527..bf50b95 100644 --- a/encoder.go +++ b/encoder.go @@ -17,7 +17,7 @@ import ( "golang.org/x/image/bmp" ) -func WebpEncoder(p1, p2 string, quality float32, Log bool, c chan int) (err error) { +func webpEncoder(p1, p2 string, quality float32, Log bool, c chan int) (err error) { // if convert fails, return error; success nil log.Debugf("target: %s with quality of %f", path.Base(p1), quality) @@ -26,11 +26,11 @@ func WebpEncoder(p1, p2 string, quality float32, Log bool, c chan int) (err erro data, err := ioutil.ReadFile(p1) if err != nil { - ChanErr(c) + chanErr(c) return } - contentType := GetFileContentType(data[:512]) + contentType := getFileContentType(data[:512]) if strings.Contains(contentType, "jpeg") { img, _ = jpeg.Decode(bytes.NewReader(data)) } else if strings.Contains(contentType, "png") { @@ -47,18 +47,18 @@ func WebpEncoder(p1, p2 string, quality float32, Log bool, c chan int) (err erro msg := "image file " + path.Base(p1) + " is corrupted or not supported" log.Debug(msg) err = errors.New(msg) - ChanErr(c) + chanErr(c) return } if err = webp.Encode(&buf, img, &webp.Options{Lossless: false, Quality: quality}); err != nil { log.Error(err) - ChanErr(c) + chanErr(c) return } if err = ioutil.WriteFile(p2, buf.Bytes(), 0644); err != nil { log.Error(err) - ChanErr(c) + chanErr(c) return } @@ -66,7 +66,7 @@ func WebpEncoder(p1, p2 string, quality float32, Log bool, c chan int) (err erro log.Info("Save to " + p2 + " ok!\n") } - ChanErr(c) + chanErr(c) return nil } diff --git a/encoder_test.go b/encoder_test.go index 848e5f3..de752ba 100644 --- a/encoder_test.go +++ b/encoder_test.go @@ -1,15 +1,15 @@ package main import ( + "github.com/stretchr/testify/assert" "io/ioutil" "os" "path/filepath" "testing" ) -//go test -v -cover encoder_test.go encoder.go helper.go +//go test -v -cover . func TestWebpEncoder(t *testing.T) { - var webp = "/tmp/test-result.webp" var target = walker() @@ -19,8 +19,24 @@ func TestWebpEncoder(t *testing.T) { } _ = os.Remove(webp) + // test error + err := webpEncoder("./pics/empty.jpg", webp, 80, true, nil) + assert.NotNil(t, err) } +func TestNonImage(t *testing.T) { + var webp = "/tmp/test-result.webp" + // test error + var err = webpEncoder("./pics/empty.jpg", webp, 80, true, nil) + assert.NotNil(t, err) +} + +func TestWriteFail(t *testing.T) { + // test permission denied + var webp = "/123.webp" + var err = webpEncoder("./pics/png.jpg", webp, 80, true, nil) + assert.NotNil(t, err) +} func walker() []string { var list []string _ = filepath.Walk("./pics", func(path string, info os.FileInfo, err error) error { @@ -34,8 +50,8 @@ func walker() []string { func runEncoder(t *testing.T, file string, webp string) { var c chan int - //t.Logf("Convert from %s to %s", file, webp) - var err = WebpEncoder(file, webp, 80, false, c) + //t.Logf("convert from %s to %s", file, webp) + var err = webpEncoder(file, webp, 80, true, c) if file == "pics/empty.jpg" && err != nil { t.Log("Empty file, that's okay.") } else if err != nil { @@ -43,7 +59,7 @@ func runEncoder(t *testing.T, file string, webp string) { } data, _ := ioutil.ReadFile(webp) - types := GetFileContentType(data[:512]) + types := getFileContentType(data[:512]) if types != "image/webp" { t.Fatal("Fatal, file type is wrong!") } diff --git a/go.mod b/go.mod index de70ada..4347c7b 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,16 @@ module webp_server_go -go 1.13 +go 1.15 require ( github.com/chai2010/webp v1.1.0 - github.com/gofiber/fiber v1.4.0 + github.com/gofiber/fiber/v2 v2.1.4 github.com/sirupsen/logrus v1.6.0 github.com/stretchr/testify v1.3.0 golang.org/x/image v0.0.0-20200119044424-58c23975cae1 ) + +replace ( + github.com/gofiber/fiber/v2 v2.1.4 => github.com/webp-sh/fiber/v2 v2.1.4 + github.com/chai2010/webp v1.1.0 => github.com/webp-sh/webp v1.1.1 +) \ No newline at end of file diff --git a/helper.go b/helper.go index 8c806e3..ef65d90 100644 --- a/helper.go +++ b/helper.go @@ -15,20 +15,20 @@ import ( log "github.com/sirupsen/logrus" ) -func ChanErr(ccc chan int) { +func chanErr(ccc chan int) { if ccc != nil { ccc <- 1 } } -func GetFileContentType(buffer []byte) string { +func getFileContentType(buffer []byte) string { // Use the net/http package's handy DectectContentType function. Always returns a valid // content-type by returning "application/octet-stream" if no others seemed to match. contentType := http.DetectContentType(buffer) return contentType } -func FileCount(dir string) int { +func fileCount(dir string) int { count := 0 _ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { @@ -40,7 +40,7 @@ func FileCount(dir string) int { return count } -func ImageExists(filename string) bool { +func imageExists(filename string) bool { info, err := os.Stat(filename) if os.IsNotExist(err) { return false @@ -51,10 +51,11 @@ func ImageExists(filename string) bool { // Check for remote filepath, e.g: https://test.webp.sh/node.png // return StatusCode, etagValue -func GetRemoteImageInfo(fileUrl string) (int, string) { +func getRemoteImageInfo(fileUrl string) (int, string) { res, err := http.Head(fileUrl) if err != nil { - log.Fatal("Connection to remote error!") + log.Errorln("Connection to remote error!") + return http.StatusInternalServerError, "" } if res.StatusCode != 404 { etagValue := res.Header.Get("etag") @@ -67,7 +68,7 @@ func GetRemoteImageInfo(fileUrl string) (int, string) { return res.StatusCode, "" } -func FetchRemoteImage(filepath string, url string) error { +func fetchRemoteImage(filepath string, url string) error { resp, err := http.Get(url) if err != nil { return err @@ -86,7 +87,7 @@ func FetchRemoteImage(filepath string, url string) error { // Given /path/to/node.png // Delete /path/to/node.png* -func CleanProxyCache(cacheImagePath string) { +func cleanProxyCache(cacheImagePath string) { // Delete /node.png* files, err := filepath.Glob(cacheImagePath + "*") if err != nil { @@ -99,11 +100,12 @@ func CleanProxyCache(cacheImagePath string) { } } -func GenWebpAbs(RawImagePath string, ExhaustPath string, ImgFilename string, reqURI string) (string, string) { +func genWebpAbs(RawImagePath string, ExhaustPath string, ImgFilename string, reqURI string) (string, string) { // get file mod time STAT, err := os.Stat(RawImagePath) if err != nil { log.Error(err.Error()) + return "", "" } ModifiedTime := STAT.ModTime().Unix() // webpFilename: abc.jpg.png -> abc.jpg.png1582558990.webp @@ -116,7 +118,7 @@ func GenWebpAbs(RawImagePath string, ExhaustPath string, ImgFilename string, req return cwd, WebpAbsolutePath } -func GenEtag(ImgAbsPath string) string { +func genEtag(ImgAbsPath string) string { data, err := ioutil.ReadFile(ImgAbsPath) if err != nil { log.Info(err) diff --git a/helper_test.go b/helper_test.go index a95133d..b272236 100644 --- a/helper_test.go +++ b/helper_test.go @@ -1,48 +1,49 @@ package main import ( + "io/ioutil" + "net/http" + "path/filepath" "strings" "testing" "github.com/stretchr/testify/assert" ) -// test this file: go test -v -cover helper_test.go helper.go -// test one function: go test -run TestGetFileContentType helper_test.go helper.go -v +// test this file: go test -v -cover . func TestGetFileContentType(t *testing.T) { var data = []byte("hello") var expected = "text/plain; charset=utf-8" - var result = GetFileContentType(data) + var result = getFileContentType(data) assert.Equalf(t, result, expected, "Result: [%s], Expected: [%s]", result, expected) } -// TODO: make a universal logging function func TestFileCount(t *testing.T) { var data = ".github" var expected = 2 - var result = FileCount(data) + var result = fileCount(data) assert.Equalf(t, result, expected, "Result: [%d], Expected: [%d]", result, expected) } func TestImageExists(t *testing.T) { var data = "./pics/empty.jpg" - var result = !ImageExists(data) + var result = !imageExists(data) if result { t.Errorf("Result: [%v], Expected: [%v]", result, false) } data = ".pics/empty2.jpg" - result = ImageExists(data) + result = imageExists(data) assert.Falsef(t, result, "Result: [%v], Expected: [%v]", result, false) } func TestGenWebpAbs(t *testing.T) { - cwd, cooked := GenWebpAbs("./pics/webp_server.png", "/tmp", + cwd, cooked := genWebpAbs("./pics/webp_server.png", "/tmp", "test", "a") if !strings.Contains(cwd, "webp_server_go") { t.Logf("Result: [%v], Expected: [%v]", cwd, "webp_server_go") @@ -56,7 +57,7 @@ func TestGenWebpAbs(t *testing.T) { func TestGenEtag(t *testing.T) { var data = "./pics/png.jpg" var expected = "W/\"1020764-262C0329\"" - var result = GenEtag(data) + var result = genEtag(data) assert.Equalf(t, result, expected, "Result: [%s], Expected: [%s]", result, expected) @@ -104,3 +105,55 @@ func TestGoOrigin(t *testing.T) { } } + +func TestChanErr(t *testing.T) { + var value = 2 + var testC = make(chan int, 2) + testC <- value + chanErr(testC) + value = <-testC + assert.Equal(t, 2, value) +} + +func TestGetRemoteImageInfo(t *testing.T) { + url := "http://github.com/favicon.ico" + statusCode, etag := getRemoteImageInfo(url) + assert.NotEqual(t, "", etag) + assert.Equal(t, statusCode, http.StatusOK) + + // test non-exist url + url = "http://sdahjajda.com" + statusCode, etag = getRemoteImageInfo(url) + assert.Equal(t, "", etag) + assert.Equal(t, statusCode, http.StatusInternalServerError) +} + +func TestFetchRemoteImage(t *testing.T) { + // test the normal one + fp := filepath.Join("./exhaust", "test.ico") + url := "http://github.com/favicon.ico" + err := fetchRemoteImage(fp, url) + assert.Equal(t, err, nil) + data, _ := ioutil.ReadFile(fp) + assert.Equal(t, "image/x-icon", getFileContentType(data)) + + // test can't create file + err = fetchRemoteImage("/", url) + assert.NotNil(t, err) + + // test bad url + err = fetchRemoteImage(fp, "http://ahjdsgdsghja.cya") + assert.NotNil(t, err) +} + +func TestCleanProxyCache(t *testing.T) { + // test normal situation + fp := filepath.Join("./exhaust", "sample.png.12345.webp") + _ = ioutil.WriteFile(fp, []byte("1234"), 0755) + assert.True(t, imageExists(fp)) + cleanProxyCache(fp) + assert.False(t, imageExists(fp)) + + // test bad dir + cleanProxyCache("/aasdyg/dhj2/dagh") +} diff --git a/prefetch.go b/prefetch.go index d257ad0..89e828f 100644 --- a/prefetch.go +++ b/prefetch.go @@ -4,15 +4,15 @@ import ( "fmt" "os" "path" - "time" "path/filepath" "strconv" "strings" + "time" log "github.com/sirupsen/logrus" ) -func PrefetchImages(confImgPath string, ExhaustPath string, QUALITY string) { +func prefetchImages(confImgPath string, ExhaustPath string, QUALITY string) { var sTime = time.Now() // maximum ongoing prefetch is depending on your core of CPU log.Infof("Prefetching using %d cores", jobs) @@ -22,7 +22,7 @@ func PrefetchImages(confImgPath string, ExhaustPath string, QUALITY string) { } //prefetch, recursive through the dir - all := FileCount(confImgPath) + all := fileCount(confImgPath) count := 0 err := filepath.Walk(confImgPath, func(picAbsPath string, info os.FileInfo, err error) error { @@ -31,10 +31,10 @@ func PrefetchImages(confImgPath string, ExhaustPath string, QUALITY string) { } // RawImagePath string, ImgFilename string, reqURI string proposedURI := strings.Replace(picAbsPath, confImgPath, "", 1) - _, p2 := GenWebpAbs(picAbsPath, ExhaustPath, info.Name(), proposedURI) + _, p2 := genWebpAbs(picAbsPath, ExhaustPath, info.Name(), proposedURI) q, _ := strconv.ParseFloat(QUALITY, 32) _ = os.MkdirAll(path.Dir(p2), 0755) - go WebpEncoder(picAbsPath, p2, float32(q), false, finishChan) + go webpEncoder(picAbsPath, p2, float32(q), false, finishChan) count += <-finishChan //progress bar _, _ = fmt.Fprintf(os.Stdout, "[Webp Server started] - convert in progress: %d/%d\r", count, all) @@ -45,6 +45,6 @@ func PrefetchImages(confImgPath string, ExhaustPath string, QUALITY string) { } elapsed := time.Since(sTime) _, _ = fmt.Fprintf(os.Stdout, "Prefetch completeY(^_^)Y\n\n") - _, _ = fmt.Fprintf(os.Stdout, "Convert %d file in %s (^_^)Y\n\n", count, elapsed) + _, _ = fmt.Fprintf(os.Stdout, "convert %d file in %s (^_^)Y\n\n", count, elapsed) } diff --git a/prefetch_test.go b/prefetch_test.go new file mode 100644 index 0000000..24e3e67 --- /dev/null +++ b/prefetch_test.go @@ -0,0 +1,29 @@ +// webp_server_go - prefetch_test.go +// 2020-11-10 09:27 +// Benny + +package main + +import ( + "github.com/stretchr/testify/assert" + "os" + "testing" +) + +func TestPrefetchImages(t *testing.T) { + // single thread + fp := "./prefetch" + _ = os.Mkdir(fp, 0755) + prefetchImages("./pics", "./prefetch", "80") + count := fileCount("./prefetch") + assert.Equal(t, 6, count) + _ = os.RemoveAll(fp) + + // concurrency + jobs = 2 + _ = os.Mkdir(fp, 0755) + prefetchImages("./pics", "./prefetch", "80") + count = fileCount("./prefetch") + assert.Equal(t, 4, count) + _ = os.RemoveAll(fp) +} diff --git a/router.go b/router.go index 1400f0d..fbafff6 100644 --- a/router.go +++ b/router.go @@ -1,159 +1,141 @@ package main import ( + "errors" "fmt" + "github.com/gofiber/fiber/v2" + log "github.com/sirupsen/logrus" "os" "path" "path/filepath" "strconv" "strings" - - log "github.com/sirupsen/logrus" - - "github.com/gofiber/fiber" ) -func Convert(ImgPath string, ExhaustPath string, AllowedTypes []string, QUALITY string, proxyMode bool) func(c *fiber.Ctx) { - return func(c *fiber.Ctx) { - //basic vars - var reqURI = c.Path() // /mypic/123.jpg - var RawImageAbs = path.Join(ImgPath, reqURI) // /home/xxx/mypic/123.jpg - var ImgFilename = path.Base(reqURI) // pure filename, 123.jpg - var finalFile string // We'll only need one c.sendFile() - var UA = c.Get("User-Agent") - done := goOrigin(UA) - if done { - log.Infof("A Safari/IE/whatever user has arrived...%s", UA) - // Check for Safari users. If they're Safari, just simply ignore everything. +func convert(c *fiber.Ctx) error { + //basic vars + var reqURI = c.Path() // /mypic/123.jpg + var rawImageAbs = path.Join(config.ImgPath, reqURI) // /home/xxx/mypic/123.jpg + var imgFilename = path.Base(reqURI) // pure filename, 123.jpg + var finalFile string // We'll only need one c.sendFile() + var UA = c.Get("User-Agent") + done := goOrigin(UA) + if done { + log.Infof("A Safari/IE/whatever user has arrived...%s", UA) + // Check for Safari users. If they're Safari, just simply ignore everything. - etag := GenEtag(RawImageAbs) - c.Set("ETag", etag) - c.SendFile(RawImageAbs) - return - } - log.Debugf("Incoming connection from %s@%s with %s", UA, c.IP(), ImgFilename) + etag := genEtag(rawImageAbs) + c.Set("ETag", etag) + return c.SendFile(rawImageAbs) + } + log.Debugf("Incoming connection from %s@%s with %s", UA, c.IP(), imgFilename) - // check ext - // TODO: may remove this function. Check in Nginx. - var allowed = false - for _, ext := range AllowedTypes { - haystack := strings.ToLower(ImgFilename) - needle := strings.ToLower("." + ext) - if strings.HasSuffix(haystack, needle) { - allowed = true - break - } else { - allowed = false - } - } - if !allowed { - msg := "File extension not allowed! " + ImgFilename - log.Warn(msg) - c.Send(msg) - if ImageExists(RawImageAbs) { - etag := GenEtag(RawImageAbs) - c.Set("ETag", etag) - c.SendFile(RawImageAbs) - } - return - } - - // Start Proxy Mode - if proxyMode { - // https://test.webp.sh/node.png - realRemoteAddr := ImgPath + reqURI - // Ping Remote for status code and etag info - - // If status code is 200 - // Check for local /node.png-etag- - // if exist - // Send local cache - // else - // Delete local /node.png* - // Fetch and convert to /node.png-etag- - // Send local cache - // else status code is 404 - // Delete /node.png* - // Send 404 - fmt.Println("Remote Addr is " + realRemoteAddr + ", fetching..") - statusCode, etagValue := GetRemoteImageInfo(realRemoteAddr) - if statusCode == 200 { - // Check local path: /node.png-etag- - localEtagImagePath := ExhaustPath + reqURI + "-etag-" + etagValue - if ImageExists(localEtagImagePath) { - c.SendFile(localEtagImagePath) - } else { - // Temporary store of remote file. - // ./remote-raw/node.png - CleanProxyCache(ExhaustPath + reqURI + "*") - localRemoteTmpPath := "./remote-raw" + reqURI - FetchRemoteImage(localRemoteTmpPath, realRemoteAddr) - q, _ := strconv.ParseFloat(QUALITY, 32) - _ = os.MkdirAll(path.Dir(localEtagImagePath), 0755) - err := WebpEncoder(localRemoteTmpPath, localEtagImagePath, float32(q), true, nil) - if err != nil { - fmt.Println(err) - } - c.SendFile(localEtagImagePath) - } - } else { - msg := fmt.Sprintf("Remote returned %d status code!", statusCode) - c.Send(msg) - log.Warn(msg) - c.SendStatus(statusCode) - CleanProxyCache(ExhaustPath + reqURI + "*") - return - } - // End Proxy Mode + // check ext + var allowed = false + for _, ext := range config.AllowedTypes { + haystack := strings.ToLower(imgFilename) + needle := strings.ToLower("." + ext) + if strings.HasSuffix(haystack, needle) { + allowed = true + break } else { - // Check the original image for existence, - if !ImageExists(RawImageAbs) { - msg := "Image not found!" - c.Send(msg) - log.Warn(msg) - c.SendStatus(404) - return - } - - _, WebpAbsPath := GenWebpAbs(RawImageAbs, ExhaustPath, ImgFilename, reqURI) - - if ImageExists(WebpAbsPath) { - finalFile = WebpAbsPath - } else { - // we don't have abc.jpg.png1582558990.webp - // delete the old pic and convert a new one. - // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp - destHalfFile := path.Clean(path.Join(WebpAbsPath, path.Dir(reqURI), ImgFilename)) - matches, err := filepath.Glob(destHalfFile + "*") - if err != nil { - log.Error(err.Error()) - } else { - // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558100.webp <- older ones will be removed - // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp <- keep the latest one - for _, p := range matches { - if strings.Compare(destHalfFile, p) != 0 { - _ = os.Remove(p) - } - } - } - - //for webp, we need to create dir first - _ = os.MkdirAll(path.Dir(WebpAbsPath), 0755) - q, _ := strconv.ParseFloat(QUALITY, 32) - err = WebpEncoder(RawImageAbs, WebpAbsPath, float32(q), true, nil) - - if err != nil { - log.Error(err) - c.SendStatus(400) - c.Send("Bad file!") - return - } - finalFile = WebpAbsPath - } - etag := GenEtag(finalFile) + allowed = false + } + } + if !allowed { + msg := "File extension not allowed! " + imgFilename + log.Warn(msg) + _ = c.Send([]byte(msg)) + if imageExists(rawImageAbs) { + etag := genEtag(rawImageAbs) c.Set("ETag", etag) - c.SendFile(finalFile) + return c.SendFile(rawImageAbs) + } + return errors.New(msg) + } + + // Start Proxy Mode + if proxyMode { + // https://test.webp.sh/node.png + realRemoteAddr := config.ImgPath + reqURI + // Ping Remote for status code and etag info + fmt.Println("Remote Addr is " + realRemoteAddr + ", fetching..") + statusCode, etagValue := getRemoteImageInfo(realRemoteAddr) + if statusCode == 200 { + // Check local path: /node.png-etag- + localEtagImagePath := config.ExhaustPath + reqURI + "-etag-" + etagValue + if imageExists(localEtagImagePath) { + return c.SendFile(localEtagImagePath) + } else { + // Temporary store of remote file. + // ./remote-raw/node.png + cleanProxyCache(config.ExhaustPath + reqURI + "*") + localRemoteTmpPath := "./remote-raw" + reqURI + _ = fetchRemoteImage(localRemoteTmpPath, realRemoteAddr) + q, _ := strconv.ParseFloat(config.Quality, 32) + _ = os.MkdirAll(path.Dir(localEtagImagePath), 0755) + err := webpEncoder(localRemoteTmpPath, localEtagImagePath, float32(q), true, nil) + if err != nil { + fmt.Println(err) + } + return c.SendFile(localEtagImagePath) + } + } else { + msg := fmt.Sprintf("Remote returned %d status code!", statusCode) + _ = c.Send([]byte(msg)) + log.Warn(msg) + _ = c.SendStatus(statusCode) + cleanProxyCache(config.ExhaustPath + reqURI + "*") + return errors.New(msg) + } + // End Proxy Mode + } else { + // Check the original image for existence, + if !imageExists(rawImageAbs) { + msg := "image not found" + _ = c.Send([]byte(msg)) + log.Warn(msg) + _ = c.SendStatus(404) + return errors.New(msg) } + _, webpAbsPath := genWebpAbs(rawImageAbs, config.ExhaustPath, imgFilename, reqURI) + + if imageExists(webpAbsPath) { + finalFile = webpAbsPath + } else { + // we don't have abc.jpg.png1582558990.webp + // delete the old pic and convert a new one. + // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp + destHalfFile := path.Clean(path.Join(webpAbsPath, path.Dir(reqURI), imgFilename)) + matches, err := filepath.Glob(destHalfFile + "*") + if err != nil { + log.Error(err.Error()) + } else { + // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558100.webp <- older ones will be removed + // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp <- keep the latest one + for _, p := range matches { + if strings.Compare(destHalfFile, p) != 0 { + _ = os.Remove(p) + } + } + } + + //for webp, we need to create dir first + err = os.MkdirAll(path.Dir(webpAbsPath), 0755) + q, _ := strconv.ParseFloat(config.Quality, 32) + err = webpEncoder(rawImageAbs, webpAbsPath, float32(q), true, nil) + + if err != nil { + log.Error(err) + _ = c.SendStatus(400) + _ = c.Send([]byte("Bad file!")) + return err + } + finalFile = webpAbsPath + } + etag := genEtag(finalFile) + c.Set("ETag", etag) + return c.SendFile(finalFile) } } diff --git a/router_test.go b/router_test.go new file mode 100644 index 0000000..fdfe3d1 --- /dev/null +++ b/router_test.go @@ -0,0 +1,125 @@ +// webp_server_go - webp-server_test +// 2020-11-09 11:55 +// Benny + +package main + +import ( + "github.com/gofiber/fiber/v2" + "github.com/stretchr/testify/assert" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" +) + +var ( + chromeUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36" + SafariUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1 Safari/605.1.15" +) + +func TestConvert(t *testing.T) { + setupParam() + var testChromeLink = map[string]string{ + "http://127.0.0.1:3333/webp_server.jpg": "image/webp", + "http://127.0.0.1:3333/webp_server.bmp": "image/webp", + "http://127.0.0.1:3333/webp_server.png": "image/webp", + "http://127.0.0.1:3333/empty.jpg": "text/plain; charset=utf-8", + "http://127.0.0.1:3333/png.jpg": "image/webp", + "http://127.0.0.1:3333/12314.jpg": "text/plain; charset=utf-8", + "http://127.0.0.1:3333/dir1/inside.jpg": "image/webp", + } + + var testSafariLink = map[string]string{ + "http://127.0.0.1:3333/webp_server.jpg": "image/jpeg", + "http://127.0.0.1:3333/webp_server.bmp": "image/bmp", + "http://127.0.0.1:3333/webp_server.png": "image/png", + "http://127.0.0.1:3333/empty.jpg": "text/plain; charset=utf-8", + "http://127.0.0.1:3333/png.jpg": "image/png", + "http://127.0.0.1:3333/12314.jpg": "text/plain; charset=utf-8", + "http://127.0.0.1:3333/dir1/inside.jpg": "image/jpeg", + } + + var app = fiber.New() + app.Get("/*", convert) + + // test Chrome + for url, respType := range testChromeLink { + _, data := requestToServer(url, app, chromeUA) + contentType := getFileContentType(data) + assert.Equal(t, respType, contentType) + } + + // test Safari + for url, respType := range testSafariLink { + _, data := requestToServer(url, app, SafariUA) + contentType := getFileContentType(data) + assert.Equal(t, respType, contentType) + } + +} + +func TestConvertNotAllowed(t *testing.T) { + setupParam() + config.AllowedTypes = []string{"jpg", "png", "jpeg"} + + var app = fiber.New() + app.Get("/*", convert) + + // not allowed, but we have the file + url := "http://127.0.0.1:3333/webp_server.bmp" + _, data := requestToServer(url, app, chromeUA) + contentType := getFileContentType(data) + assert.Equal(t, "image/bmp", contentType) + + // not allowed, random file + url = url + "hagdgd" + _, data = requestToServer(url, app, chromeUA) + assert.Contains(t, string(data), "File extension not allowed") + +} + +func TestConvertProxyModeBad(t *testing.T) { + setupParam() + proxyMode = true + + var app = fiber.New() + app.Get("/*", convert) + + // this is local image, should be 500 + url := "http://127.0.0.1:3333/webp_server.bmp" + resp, _ := requestToServer(url, app, chromeUA) + assert.Equal(t, http.StatusInternalServerError, resp.StatusCode) + +} + +func TestConvertProxyModeWork(t *testing.T) { + setupParam() + proxyMode = true + + var app = fiber.New() + app.Get("/*", convert) + + config.ImgPath = "https://webp.sh" + url := "https://webp.sh/images/cover.jpg" + + resp, data := requestToServer(url, app, chromeUA) + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, "image/webp", getFileContentType(data)) + +} + +func setupParam() { + // setup parameters here... + config.ImgPath = "./pics" + config.ExhaustPath = "./exhaust" + config.AllowedTypes = []string{"jpg", "png", "jpeg", "bmp"} +} + +func requestToServer(url string, app *fiber.App, ua string) (*http.Response, []byte) { + req := httptest.NewRequest("GET", url, nil) + req.Header.Set("User-Agent", ua) + resp, _ := app.Test(req) + data, _ := ioutil.ReadAll(resp.Body) + return resp, data +} diff --git a/scripts/unit_test.sh b/scripts/unit_test.sh deleted file mode 100644 index 2900710..0000000 --- a/scripts/unit_test.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -# bash scripts/unit_test.sh -# check $? for success or failure -go test -v -cover encoder_test.go encoder.go helper.go -go test -v -cover helper_test.go helper.go - -# if [[ $? -ne 0 ]] ; then -# echo "TEST FAILED!!! PLEASE DOUBLE CHECK." -# fi diff --git a/scripts/unix.sh b/scripts/unix.sh deleted file mode 100644 index c915834..0000000 --- a/scripts/unix.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -cd .. -git pull -platform=$(uname -a) - -if [[ $platform =~ "Darwin" ]] -then - go build -o webp-server-darwin-amd64 webp-server.go -elif [[ $platform =~ "x86_64" ]];then - go build -o webp-server-unix-amd64 webp-server.go -else - go build -o webp-server-linux-amd64 webp-server.go -fi diff --git a/scripts/webp_server.spec b/scripts/webp_server.spec index b6e5613..9a8d61d 100644 --- a/scripts/webp_server.spec +++ b/scripts/webp_server.spec @@ -1,5 +1,5 @@ Name: webp-server -Version: 0.1.2 +Version: 0.2.1 Release: 1%{?dist} Summary: Go version of WebP Server. A tool that will serve your JPG/PNGs as WebP format with compression, on-the-fly. diff --git a/init/webps.service b/scripts/webps.service similarity index 100% rename from init/webps.service rename to scripts/webps.service diff --git a/scripts/windows.bat b/scripts/windows.bat deleted file mode 100644 index 6cc15b6..0000000 --- a/scripts/windows.bat +++ /dev/null @@ -1,14 +0,0 @@ -cd .. -git pull - -IF EXIST "%PROGRAMFILES(X86)%" (GOTO 64BIT) ELSE (GOTO 32BIT) -:64BIT -go build -o webp-server-windows-amd64.exe webp-server.go -GOTO END - -:32BIT -echo 32-bit... -go build -o webp-server-windows-i386.exe webp-server.go -GOTO END - -pause diff --git a/update.go b/update.go index 033b292..e7ce14e 100644 --- a/update.go +++ b/update.go @@ -50,7 +50,6 @@ func autoUpdate() { } data, _ := ioutil.ReadAll(resp.Body) _ = os.Mkdir("update", 0755) - // TODO: checksum err := ioutil.WriteFile(path.Join("update", filename), data, 0755) if err == nil { diff --git a/update_test.go b/update_test.go new file mode 100644 index 0000000..e50cc12 --- /dev/null +++ b/update_test.go @@ -0,0 +1,27 @@ +// webp_server_go - update_test +// 2020-11-10 09:36 +// Benny + +package main + +import ( + "github.com/stretchr/testify/assert" + "os" + "testing" +) + +func TestNormalAutoUpdate(t *testing.T) { + version = "0.0.1" + dir := "./update" + autoUpdate() + assert.NotEqual(t, 0, fileCount(dir)) + _ = os.RemoveAll(dir) +} + +func TestNoNeedAutoUpdate(t *testing.T) { + version = "99.99" + dir := "./update" + autoUpdate() + info, _ := os.Stat(dir) + assert.Nil(t, info) +} diff --git a/webp-server.go b/webp-server.go index aa716ae..0f7c759 100644 --- a/webp-server.go +++ b/webp-server.go @@ -5,42 +5,44 @@ import ( "flag" "fmt" "os" - "path" "regexp" "runtime" - "github.com/gofiber/fiber" + "github.com/gofiber/fiber/v2" log "github.com/sirupsen/logrus" ) type Config struct { - HOST string - PORT string - ImgPath string `json:"IMG_PATH"` - QUALITY string + Host string `json:"HOST"` + Port string `json:"PORT"` + ImgPath string `json:"IMG_PATH"` + Quality string `json:"QUALITY"` AllowedTypes []string `json:"ALLOWED_TYPES"` ExhaustPath string `json:"EXHAUST_PATH"` } -const version = "0.2.0" +var ( + configPath string + jobs int + dumpConfig, dumpSystemd, verboseMode, prefetch, showVersion bool -var configPath string -var prefetch bool -var jobs int -var dumpConfig bool -var dumpSystemd bool -var verboseMode bool + proxyMode bool + config Config + version = "0.2.1" +) -const sampleConfig = ` +const ( + sampleConfig = ` { - "HOST": "127.0.0.1", - "PORT": "3333", - "QUALITY": "80", - "IMG_PATH": "/path/to/pics", - "EXHAUST_PATH": "", - "ALLOWED_TYPES": ["jpg","png","jpeg","bmp"] + "HOST": "127.0.0.1", + "PORT": "3333", + "QUALITY": "80", + "IMG_PATH": "./pics", + "EXHAUST_PATH": "./exhaust", + "ALLOWED_TYPES": ["jpg","png","jpeg","bmp"] }` -const sampleSystemd = ` + + sampleSystemd = ` [Unit] Description=WebP Server Go Documentation=https://github.com/webp-sh/webp_server_go @@ -56,26 +58,27 @@ RestartSec=3s [Install] WantedBy=multi-user.target` +) func loadConfig(path string) Config { - var config Config jsonObject, err := os.Open(path) if err != nil { log.Fatal(err) } - defer jsonObject.Close() decoder := json.NewDecoder(jsonObject) _ = decoder.Decode(&config) + _ = jsonObject.Close() return config } -func init() { +func deferInit() { flag.StringVar(&configPath, "config", "config.json", "/path/to/config.json. (Default: ./config.json)") flag.BoolVar(&prefetch, "prefetch", false, "Prefetch and convert image to webp") flag.IntVar(&jobs, "jobs", runtime.NumCPU(), "Prefetch thread, default is all.") flag.BoolVar(&dumpConfig, "dump-config", false, "Print sample config.json") flag.BoolVar(&dumpSystemd, "dump-systemd", false, "Print sample systemd service file.") flag.BoolVar(&verboseMode, "v", false, "Verbose, print out debug info.") + flag.BoolVar(&showVersion, "V", false, "Show version information.") flag.Parse() // Logrus log.SetOutput(os.Stdout) @@ -99,6 +102,17 @@ func init() { } func main() { + // Our banner + banner := fmt.Sprintf(` +▌ ▌ ▌ ▛▀▖ ▞▀▖ ▞▀▖ +▌▖▌▞▀▖▛▀▖▙▄▘ ▚▄ ▞▀▖▙▀▖▌ ▌▞▀▖▙▀▖ ▌▄▖▞▀▖ +▙▚▌▛▀ ▌ ▌▌ ▖ ▌▛▀ ▌ ▐▐ ▛▀ ▌ ▌ ▌▌ ▌ +▘ ▘▝▀▘▀▀ ▘ ▝▀ ▝▀▘▘ ▘ ▝▀▘▘ ▝▀ ▝▀ + +Webp Server Go - v%s +Develop by WebP Server team. https://github.com/webp-sh`, version) + + deferInit() // process cli params if dumpConfig { fmt.Println(sampleConfig) @@ -108,49 +122,40 @@ func main() { fmt.Println(sampleSystemd) os.Exit(0) } + if showVersion { + fmt.Printf("\n %c[1;32m%s%c[0m\n\n", 0x1B, banner+"", 0x1B) + os.Exit(0) + } go autoUpdate() - config := loadConfig(configPath) + config = loadConfig(configPath) - HOST := config.HOST - PORT := config.PORT // Check for remote address matched, _ := regexp.MatchString(`^https?://`, config.ImgPath) - proxyMode := false - confImgPath := "" + proxyMode = false if matched { proxyMode = true - confImgPath = config.ImgPath } else { _, err := os.Stat(config.ImgPath) if err != nil { log.Fatalf("Your image path %s is incorrect.Please check and confirm.", config.ImgPath) } - confImgPath = path.Clean(config.ImgPath) - } - QUALITY := config.QUALITY - AllowedTypes := config.AllowedTypes - var ExhaustPath string - if len(config.ExhaustPath) == 0 { - ExhaustPath = "./exhaust" - } else { - ExhaustPath = config.ExhaustPath } if prefetch { - go PrefetchImages(confImgPath, ExhaustPath, QUALITY) + go prefetchImages(config.ImgPath, config.ExhaustPath, config.Quality) } - app := fiber.New() - app.Banner = false - app.Server = "WebP Server Go" + app := fiber.New(fiber.Config{ + ServerHeader: "Webp-Server-Go", + DisableStartupMessage: true, + }) + listenAddress := config.Host + ":" + config.Port + app.Get("/*", convert) - ListenAddress := HOST + ":" + PORT + fmt.Printf("\n %c[1;32m%s%c[0m\n\n", 0x1B, banner, 0x1B) + fmt.Println("Webp-Server-Go is Running on http://" + listenAddress) - // Server Info - log.Infof("WebP Server %s %s", version, ListenAddress) - - app.Get("/*", Convert(confImgPath, ExhaustPath, AllowedTypes, QUALITY, proxyMode)) - app.Listen(ListenAddress) + _ = app.Listen(listenAddress) } diff --git a/webp-server_test.go b/webp-server_test.go new file mode 100644 index 0000000..d4565df --- /dev/null +++ b/webp-server_test.go @@ -0,0 +1,50 @@ +// webp_server_go - webp-server_test +// 2020-11-10 09:41 +// Benny + +package main + +import ( + "github.com/stretchr/testify/assert" + "net" + "runtime" + "testing" + "time" +) + +// due to test limit, we can't test for cli param part. + +func TestLoadConfig(t *testing.T) { + c := loadConfig("./config.json") + assert.Equal(t, "./exhaust", c.ExhaustPath) + assert.Equal(t, "127.0.0.1", c.Host) + assert.Equal(t, "3333", c.Port) + assert.Equal(t, "80", c.Quality) + assert.Equal(t, "./pics", c.ImgPath) + assert.Equal(t, []string{"jpg", "png", "jpeg", "bmp"}, c.AllowedTypes) +} + +func TestDeferInit(t *testing.T) { + // test initial value + assert.Equal(t, "", configPath) + assert.False(t, prefetch) + assert.Equal(t, false, dumpSystemd) + assert.Equal(t, false, dumpConfig) + assert.False(t, verboseMode) +} + +func TestMainFunction(t *testing.T) { + go main() + time.Sleep(time.Second * 2) + // test read config value + assert.Equal(t, "config.json", configPath) + assert.False(t, prefetch) + assert.Equal(t, runtime.NumCPU(), jobs) + assert.Equal(t, false, dumpSystemd) + assert.Equal(t, false, dumpConfig) + assert.False(t, verboseMode) + // test port + conn, err := net.DialTimeout("tcp", net.JoinHostPort("127.0.0.1", "3333"), time.Second*2) + assert.Nil(t, err) + assert.NotNil(t, conn) +}