diff --git a/config.go b/config.go index 2c30063..706aa8d 100644 --- a/config.go +++ b/config.go @@ -22,11 +22,7 @@ var ( remoteRaw = "remote-raw" config Config version = "0.3.2" -) - -const ( - NotCompressed = "not_compressed" - WebpBigger = "webp_bigger" + releaseUrl = "https://github.com/webp-sh/webp_server_go/releases/latest/download/" ) const ( diff --git a/encoder_test.go b/encoder_test.go index ad49257..693baf2 100644 --- a/encoder_test.go +++ b/encoder_test.go @@ -8,7 +8,17 @@ import ( "testing" ) -//go test -v -cover . +func walker() []string { + var list []string + _ = filepath.Walk("./pics", func(path string, info os.FileInfo, err error) error { + if !info.IsDir() { + list = append(list, path) + } + return nil + }) + return list +} + func TestWebpEncoder(t *testing.T) { var webp = "/tmp/test-result.webp" var target = walker() @@ -25,24 +35,18 @@ func TestWebpEncoder(t *testing.T) { _ = os.Remove(webp) } -func TestNonImage(t *testing.T) { +func TestNonExistImage(t *testing.T) { var webp = "/tmp/test-result.webp" // test error var err = webpEncoder("./pics/empty.jpg", webp, 80, true, nil) assert.NotNil(t, err) _ = os.Remove(webp) - } -func walker() []string { - var list []string - _ = filepath.Walk("./pics", func(path string, info os.FileInfo, err error) error { - if !info.IsDir() { - list = append(list, path) - } - return nil - }) - return list +func TestConvertFail(t *testing.T) { + var webp = "/tmp/test-result.webp" + var err = webpEncoder("./pics/webp_server.jpg", webp, -1, true, nil) + assert.NotNil(t, t, err) } func runEncoder(t *testing.T, file string, webp string) { diff --git a/helper.go b/helper.go index 88a9372..7a9eb5f 100644 --- a/helper.go +++ b/helper.go @@ -9,6 +9,7 @@ import ( "os" "path" "path/filepath" + "strconv" "strings" @@ -32,6 +33,9 @@ func fileCount(dir string) int { count := 0 _ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } if !info.IsDir() { count += 1 } @@ -50,22 +54,24 @@ func imageExists(filename string) bool { } // Check for remote filepath, e.g: https://test.webp.sh/node.png -// return StatusCode, etagValue -func getRemoteImageInfo(fileUrl string) (int, string) { +// return StatusCode, etagValue and length +func getRemoteImageInfo(fileUrl string) (int, string, string) { res, err := http.Head(fileUrl) if err != nil { log.Errorln("Connection to remote error!") - return http.StatusInternalServerError, "" + return http.StatusInternalServerError, "", "" } if res.StatusCode != 404 { etagValue := res.Header.Get("etag") if etagValue == "" { log.Info("Remote didn't return etag in header, please check.") } else { - return 200, etagValue + return 200, etagValue, res.Header.Get("content-length") } } - return res.StatusCode, "" + + return res.StatusCode, "", res.Header.Get("content-length") + } func fetchRemoteImage(filepath string, url string) error { @@ -184,3 +190,23 @@ func headerOrigin(header string) bool { return true } } + +func chooseProxy(proxyRawSize string, webpAbsPath string) bool { + var proxyRaw, _ = strconv.Atoi(proxyRawSize) + webp, _ := ioutil.ReadFile(webpAbsPath) + if len(webp) > proxyRaw { + return true + } else { + return false + } +} + +func chooseLocalSmallerFile(rawImageAbs, webpAbsPath string) string { + raw, _ := ioutil.ReadFile(rawImageAbs) + webp, _ := ioutil.ReadFile(webpAbsPath) + if len(webp) > len(raw) { + return rawImageAbs + } else { + return webpAbsPath + } +} diff --git a/helper_test.go b/helper_test.go index 309e73c..8af14f5 100644 --- a/helper_test.go +++ b/helper_test.go @@ -177,14 +177,16 @@ func TestChanErr(t *testing.T) { func TestGetRemoteImageInfo(t *testing.T) { url := "https://github.com/favicon.ico" - statusCode, etag := getRemoteImageInfo(url) + statusCode, etag, length := getRemoteImageInfo(url) assert.NotEqual(t, "", etag) + assert.NotEqual(t, "0", length) assert.Equal(t, statusCode, http.StatusOK) // test non-exist url url = "http://sdahjajda.com" - statusCode, etag = getRemoteImageInfo(url) + statusCode, etag, length = getRemoteImageInfo(url) assert.Equal(t, "", etag) + assert.Equal(t, "", length) assert.Equal(t, statusCode, http.StatusInternalServerError) } diff --git a/pics/big.jpg b/pics/big.jpg new file mode 100644 index 0000000..46c6c38 Binary files /dev/null and b/pics/big.jpg differ diff --git a/prefetch_test.go b/prefetch_test.go index 4fea632..4a134ab 100644 --- a/prefetch_test.go +++ b/prefetch_test.go @@ -16,7 +16,7 @@ func TestPrefetchImages(t *testing.T) { _ = os.Mkdir(fp, 0755) prefetchImages("./pics", "./prefetch", "80") count := fileCount("./prefetch") - assert.Equal(t, 7, count) + assert.Equal(t, 8, count) _ = os.RemoveAll(fp) // concurrency @@ -24,6 +24,11 @@ func TestPrefetchImages(t *testing.T) { _ = os.Mkdir(fp, 0755) prefetchImages("./pics", "./prefetch", "80") count = fileCount("./prefetch") - assert.Equal(t, 5, count) + assert.Equal(t, 6, count) _ = os.RemoveAll(fp) } + +func TestBadPrefetch(t *testing.T) { + jobs = 1 + prefetchImages("./pics2", "./prefetch", "80") +} diff --git a/router.go b/router.go index c6f8b0e..e2e062b 100644 --- a/router.go +++ b/router.go @@ -33,7 +33,6 @@ func convert(c *fiber.Ctx) error { if needOrigin { log.Debugf("A Safari/IE/whatever user has arrived...%s", ua) c.Set("ETag", genEtag(rawImageAbs)) - c.Set("X-Compression-Rate", NotCompressed) if proxyMode { localRemoteTmpPath := remoteRaw + reqURI _ = fetchRemoteImage(localRemoteTmpPath, rawImageAbs) @@ -69,88 +68,94 @@ func convert(c *fiber.Ctx) error { } } - // Start Proxy Mode if proxyMode { - // https://test.webp.sh/node.png - realRemoteAddr := config.ImgPath + reqURI - // Ping Remote for status code and etag info - log.Infof("Remote Addr is %s fetching", realRemoteAddr) - statusCode, etagValue := getRemoteImageInfo(realRemoteAddr) - if statusCode == 200 { - // Check local path: /node.png-etag- - localEtagImagePath := config.ExhaustPath + reqURI + "-etag-" + etagValue - if imageExists(localEtagImagePath) { - return c.SendFile(localEtagImagePath) - } else { - // Temporary store of remote file. - cleanProxyCache(config.ExhaustPath + reqURI + "*") - localRemoteTmpPath := remoteRaw + reqURI - _ = fetchRemoteImage(localRemoteTmpPath, realRemoteAddr) - q, _ := strconv.ParseFloat(config.Quality, 32) - _ = os.MkdirAll(path.Dir(localEtagImagePath), 0755) - err := webpEncoder(localRemoteTmpPath, localEtagImagePath, float32(q), true, nil) - if err != nil { - log.Warning(err) - } - return c.SendFile(localEtagImagePath) - } - } else { - msg := fmt.Sprintf("Remote returned %d status code!", statusCode) - _ = c.Send([]byte(msg)) - log.Warn(msg) - _ = c.SendStatus(statusCode) - cleanProxyCache(config.ExhaustPath + reqURI + "*") - return errors.New(msg) - } - // End Proxy Mode + return proxyHandler(c, reqURI) + } + + // Check the original image for existence, + if !imageExists(rawImageAbs) { + msg := "image not found" + _ = c.Send([]byte(msg)) + log.Warn(msg) + _ = c.SendStatus(404) + return errors.New(msg) + } + + _, webpAbsPath := genWebpAbs(rawImageAbs, config.ExhaustPath, imgFilename, reqURI) + + if imageExists(webpAbsPath) { + finalFile = webpAbsPath } else { - // Check the original image for existence, - if !imageExists(rawImageAbs) { - msg := "image not found" - _ = c.Send([]byte(msg)) - log.Warn(msg) - _ = c.SendStatus(404) - return errors.New(msg) - } - - _, webpAbsPath := genWebpAbs(rawImageAbs, config.ExhaustPath, imgFilename, reqURI) - - if imageExists(webpAbsPath) { - finalFile = webpAbsPath + // we don't have abc.jpg.png1582558990.webp + // delete the old pic and convert a new one. + // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp + destHalfFile := path.Clean(path.Join(webpAbsPath, path.Dir(reqURI), imgFilename)) + matches, err := filepath.Glob(destHalfFile + "*") + if err != nil { + log.Error(err.Error()) } else { - // we don't have abc.jpg.png1582558990.webp - // delete the old pic and convert a new one. - // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp - destHalfFile := path.Clean(path.Join(webpAbsPath, path.Dir(reqURI), imgFilename)) - matches, err := filepath.Glob(destHalfFile + "*") - if err != nil { - log.Error(err.Error()) - } else { - // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558100.webp <- older ones will be removed - // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp <- keep the latest one - for _, p := range matches { - if strings.Compare(destHalfFile, p) != 0 { - _ = os.Remove(p) - } + // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558100.webp <- older ones will be removed + // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp <- keep the latest one + for _, p := range matches { + if strings.Compare(destHalfFile, p) != 0 { + _ = os.Remove(p) } } - - //for webp, we need to create dir first - err = os.MkdirAll(path.Dir(webpAbsPath), 0755) - q, _ := strconv.ParseFloat(config.Quality, 32) - err = webpEncoder(rawImageAbs, webpAbsPath, float32(q), true, nil) - - if err != nil { - log.Error(err) - _ = c.SendStatus(400) - _ = c.Send([]byte("Bad file. " + err.Error())) - return err - } - finalFile = webpAbsPath } - etag := genEtag(finalFile) - c.Set("ETag", etag) - c.Set("X-Compression-Rate", getCompressionRate(rawImageAbs, webpAbsPath)) - return c.SendFile(finalFile) + + //for webp, we need to create dir first + err = os.MkdirAll(path.Dir(webpAbsPath), 0755) + q, _ := strconv.ParseFloat(config.Quality, 32) + err = webpEncoder(rawImageAbs, webpAbsPath, float32(q), true, nil) + + if err != nil { + log.Error(err) + _ = c.SendStatus(400) + _ = c.Send([]byte("Bad file. " + err.Error())) + return err + } + finalFile = webpAbsPath + } + etag := genEtag(finalFile) + c.Set("ETag", etag) + c.Set("X-Compression-Rate", getCompressionRate(rawImageAbs, webpAbsPath)) + finalFile = chooseLocalSmallerFile(rawImageAbs, webpAbsPath) + return c.SendFile(finalFile) + +} + +func proxyHandler(c *fiber.Ctx, reqURI string) error { + // https://test.webp.sh/node.png + realRemoteAddr := config.ImgPath + reqURI + // Ping Remote for status code and etag info + log.Infof("Remote Addr is %s fetching", realRemoteAddr) + statusCode, etagValue, remoteLength := getRemoteImageInfo(realRemoteAddr) + if statusCode == 200 { + // Check local path: /node.png-etag- + localEtagWebPPath := config.ExhaustPath + reqURI + "-etag-" + etagValue + if imageExists(localEtagWebPPath) { + chooseProxy(remoteLength, localEtagWebPPath) + return c.SendFile(localEtagWebPPath) + } else { + // Temporary store of remote file. + cleanProxyCache(config.ExhaustPath + reqURI + "*") + localRawImagePath := remoteRaw + reqURI + _ = fetchRemoteImage(localRawImagePath, realRemoteAddr) + q, _ := strconv.ParseFloat(config.Quality, 32) + _ = os.MkdirAll(path.Dir(localEtagWebPPath), 0755) + err := webpEncoder(localRawImagePath, localEtagWebPPath, float32(q), true, nil) + if err != nil { + log.Warning(err) + } + chooseProxy(remoteLength, localEtagWebPPath) + return c.SendFile(localEtagWebPPath) + } + } else { + msg := fmt.Sprintf("Remote returned %d status code!", statusCode) + _ = c.Send([]byte(msg)) + log.Warn(msg) + _ = c.SendStatus(statusCode) + cleanProxyCache(config.ExhaustPath + reqURI + "*") + return errors.New(msg) } } diff --git a/router_test.go b/router_test.go index 67e3032..10a3eea 100644 --- a/router_test.go +++ b/router_test.go @@ -10,6 +10,7 @@ import ( "io/ioutil" "net/http" "net/http/httptest" + "os" "testing" ) @@ -21,7 +22,7 @@ var ( func setupParam() { // setup parameters here... config.ImgPath = "./pics" - config.ExhaustPath = "./exhaust" + config.ExhaustPath = "./exhaust_test" config.AllowedTypes = []string{"jpg", "png", "jpeg", "bmp"} proxyMode = false @@ -55,7 +56,6 @@ func TestServerHeaders(t *testing.T) { ratio = response.Header.Get("X-Compression-Rate") etag = response.Header.Get("Etag") - assert.Equal(t, NotCompressed, ratio) assert.NotEqual(t, "", etag) } @@ -156,3 +156,20 @@ func TestConvertProxyModeWork(t *testing.T) { assert.Equal(t, http.StatusOK, resp.StatusCode) assert.Equal(t, "image/jpeg", getFileContentType(data)) } + +func TestConvertBigger(t *testing.T) { + setupParam() + config.Quality = "100" + + jpg, _ := ioutil.ReadFile("pics/big.jpg") + + var app = fiber.New() + app.Get("/*", convert) + + url := "http://127.0.0.1:3333/big.jpg" + response, data := requestToServer(url, app, chromeUA) + assert.Equal(t, "image/jpeg", response.Header.Get("content-type")) + assert.True(t, len(data) == len(jpg)) + + _ = os.RemoveAll(config.ExhaustPath) +} diff --git a/update.go b/update.go index 1a1afab..ee73861 100644 --- a/update.go +++ b/update.go @@ -41,9 +41,8 @@ func autoUpdate() { if runtime.GOOS == "windows" { filename += ".exe" } - var releaseUrl = "https://github.com/webp-sh/webp_server_go/releases/latest/download/" + filename log.Info("Downloading binary to update...") - resp, _ := http.Get(releaseUrl) + resp, _ := http.Get(releaseUrl + filename) if resp.StatusCode != 200 { log.Debugf("%s-%s not found on release.", runtime.GOOS, runtime.GOARCH) return diff --git a/update_test.go b/update_test.go index e50cc12..e757999 100644 --- a/update_test.go +++ b/update_test.go @@ -18,6 +18,15 @@ func TestNormalAutoUpdate(t *testing.T) { _ = os.RemoveAll(dir) } +func Test404AutoUpdate(t *testing.T) { + version = "0.0.1" + dir := "./update" + releaseUrl = releaseUrl + "a" + autoUpdate() + assert.Equal(t, 0, fileCount(dir)) + _ = os.RemoveAll(dir) +} + func TestNoNeedAutoUpdate(t *testing.T) { version = "99.99" dir := "./update"