From e9543522ae89191bf6266d5cd2c9cc83ffbcca12 Mon Sep 17 00:00:00 2001 From: BennyThink Date: Mon, 29 Nov 2021 17:22:27 +0800 Subject: [PATCH] avif support is almost complete --- config.go | 2 +- encoder.go | 133 ++++++++++++++++++++++++++++++++++++---------- go.mod | 4 ++ go.sum | 26 ++++++++- helper.go | 141 ++++++++++++++++++++++++++----------------------- helper_test.go | 2 +- prefetch.go | 34 ++++++------ router.go | 70 ++++++++---------------- webp-server.go | 4 +- 9 files changed, 253 insertions(+), 163 deletions(-) diff --git a/config.go b/config.go index 706aa8d..f9270a9 100644 --- a/config.go +++ b/config.go @@ -8,7 +8,7 @@ type Config struct { Host string `json:"HOST"` Port string `json:"PORT"` ImgPath string `json:"IMG_PATH"` - Quality string `json:"QUALITY"` + Quality float32 `json:"QUALITY,string"` AllowedTypes []string `json:"ALLOWED_TYPES"` ExhaustPath string `json:"EXHAUST_PATH"` } diff --git a/encoder.go b/encoder.go index f98ad25..2bf537c 100644 --- a/encoder.go +++ b/encoder.go @@ -4,28 +4,71 @@ import ( "bytes" "errors" "image" - "image/gif" "image/jpeg" "image/png" "io/ioutil" + "os" "path" + "path/filepath" "strings" log "github.com/sirupsen/logrus" + "github.com/Kagami/go-avif" "github.com/chai2010/webp" "golang.org/x/image/bmp" ) -func webpEncoder(p1, p2 string, quality float32) (err error) { - // if convert fails, return error; success nil - log.Debugf("target: %s with quality of %f", path.Base(p1), quality) - var buf bytes.Buffer - var img image.Image +func convertFilter(raw, avifPath, webpPath string, c chan int) { + // all absolute paths - data, err := ioutil.ReadFile(p1) + if !imageExists(avifPath) { + convertImage(raw, avifPath, "avif") + } + if !imageExists(webpPath) { + convertImage(raw, webpPath, "webp") + } + + if c != nil { + c <- 1 + } +} + +func convertImage(raw, optimized, itype string) { + // we don't have abc.jpg.png1582558990.webp + // delete the old pic and convert a new one. + // optimized: /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp + // we'll delete file starts with /home/webp_server/exhaust/path/to/tsuki.jpg.ts.itype + + s := strings.Split(path.Base(optimized), ".") + pattern := path.Join(path.Dir(optimized), s[0]+"."+s[1]+".*."+s[len(s)-1]) + + matches, err := filepath.Glob(pattern) if err != nil { - return + log.Error(err.Error()) + } else { + for _, p := range matches { + _ = os.Remove(p) + } + } + + //we need to create dir first + err = os.MkdirAll(path.Dir(optimized), 0755) + //q, _ := strconv.ParseFloat(config.Quality, 32) + + switch itype { + case "webp": + webpEncoder(raw, optimized, config.Quality) + case "avif": + avifEncoder(raw, optimized, config.Quality) + } + +} + +func readRawImage(imgPath string) (img image.Image, err error) { + data, err := ioutil.ReadFile(imgPath) + if err != nil { + log.Errorln(err) } contentType := getFileContentType(data[:512]) @@ -35,28 +78,64 @@ func webpEncoder(p1, p2 string, quality float32) (err error) { img, _ = png.Decode(bytes.NewReader(data)) } else if strings.Contains(contentType, "bmp") { img, _ = bmp.Decode(bytes.NewReader(data)) - } else if strings.Contains(contentType, "gif") { - // TODO: need to support animated webp - log.Warn("Gif support is not perfect!") - img, _ = gif.Decode(bytes.NewReader(data)) } if img == nil { - msg := "image file " + path.Base(p1) + " is corrupted or not supported" - log.Debug(msg) - return errors.New(msg) + errinfo := "image file " + path.Base(imgPath) + " is corrupted or not supported" + log.Errorln(errinfo) + return nil, errors.New(errinfo) } - if err = webp.Encode(&buf, img, &webp.Options{Lossless: false, Quality: quality}); err != nil { - log.Error(err) - return - } - if err = ioutil.WriteFile(p2, buf.Bytes(), 0644); err != nil { - log.Error(err) - return - } - - log.Info("Save to " + p2 + " ok!\n") - - return nil + return img, nil +} + +func avifEncoder(p1, p2 string, quality float32) { + var img image.Image + dst, err := os.Create(p2) + if err != nil { + log.Fatalf("Can't create destination file: %v", err) + } + img, err = readRawImage(p1) + if err != nil { + return + } + + var avifQuality = int((100 - quality) / 100 * avif.MaxQuality) + err = avif.Encode(dst, img, &avif.Options{Quality: avifQuality}) + if err != nil { + log.Fatalf("Can't encode source image: %v", err) + } + + convertLog("AVIF", p1, p2, quality) +} + +func webpEncoder(p1, p2 string, quality float32) { + // if convert fails, return error; success nil + var buf bytes.Buffer + var img image.Image + + img, err := readRawImage(p1) + if err != nil { + return + } + + if err := webp.Encode(&buf, img, &webp.Options{Lossless: false, Quality: quality}); err != nil { + log.Error(err) + return + } + + if err := ioutil.WriteFile(p2, buf.Bytes(), 0644); err != nil { + log.Error(err) + return + } + + convertLog("WebP", p1, p2, quality) + +} + +func convertLog(itype, p1 string, p2 string, quality float32) { + oldf, _ := os.Stat(p1) + newf, _ := os.Stat(p2) + log.Infof("%s@%.2f%%: %s->%s %d->%d %.2f%% deflated", itype, quality, + p1, p2, oldf.Size(), newf.Size(), float32(newf.Size())/float32(oldf.Size())*100) } diff --git a/go.mod b/go.mod index efb0327..493c1b1 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,15 @@ module webp_server_go go 1.15 require ( + github.com/Kagami/go-avif v0.1.0 github.com/chai2010/webp v1.1.0 github.com/gofiber/fiber/v2 v2.4.0 + github.com/h2non/filetype v1.1.3 + github.com/schollz/progressbar/v3 v3.8.3 github.com/sirupsen/logrus v1.6.0 github.com/staktrace/go-update v0.0.0-20210525161054-fc019945f9a2 github.com/stretchr/testify v1.3.0 + github.com/valyala/fasthttp v1.18.0 golang.org/x/image v0.0.0-20200119044424-58c23975cae1 ) diff --git a/go.sum b/go.sum index 4e9f542..dcfd835 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,28 @@ +github.com/Kagami/go-avif v0.1.0 h1:8GHAGLxCdFfhpd4Zg8j1EqO7rtcQNenxIDerC/uu68w= +github.com/Kagami/go-avif v0.1.0/go.mod h1:OPmPqzNdQq3+sXm0HqaUJQ9W/4k+Elbc3RSfJUemDKA= github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= +github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= +github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/klauspost/compress v1.10.7 h1:7rix8v8GpI3ZBb0nSozFRgbtXKv+hOe+qfEpZqybrAg= github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/schollz/progressbar/v3 v3.8.3 h1:FnLGl3ewlDUP+YdSwveXBaXs053Mem/du+wr7XSYKl8= +github.com/schollz/progressbar/v3 v3.8.3/go.mod h1:pWnVCjSBZsT2X3nx9HfRdnCDrpbevliMeoEVhStwHko= github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/staktrace/go-update v0.0.0-20210525161054-fc019945f9a2 h1:kyTDvRL8TyTHOx0aK1PKMPVfkI7rCLDC1nrKF7SYOBc= @@ -29,16 +43,26 @@ github.com/webp-sh/webp v1.2.0 h1:WiAR1M4Cz50Ehv0vP8VaBZwuTqBgLxcGMaBV23zLVDA= github.com/webp-sh/webp v1.2.0/go.mod h1:DWmBXPtpA/zfTgEgWxAlsER3B7nXFvDtCi1YY+K5N9w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/image v0.0.0-20200119044424-58c23975cae1 h1:5h3ngYt7+vXCDZCup/HkCQgW5XwmSvR/nA2JmJ0RErg= golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201210223839-7e3030f88018 h1:XKi8B/gRBuTZN1vU9gFsLMm6zVz5FSCDzm8JYACnjy8= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201210223839-7e3030f88018/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0 h1:xrCZDmdtoloIiooiA9q0OQb9r8HejIHYoHGhGCe1pGg= +golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/helper.go b/helper.go index 7a9eb5f..75c2c09 100644 --- a/helper.go +++ b/helper.go @@ -1,7 +1,9 @@ package main import ( + "bytes" "fmt" + "github.com/valyala/fasthttp" "hash/crc32" "io" "io/ioutil" @@ -11,26 +13,32 @@ import ( "path/filepath" "strconv" - "strings" - + "github.com/h2non/filetype" log "github.com/sirupsen/logrus" + "strings" ) -func chanErr(ccc chan int) { - if ccc != nil { - ccc <- 1 - } +func avifMatcher(buf []byte) bool { + // 0000001C 66747970 6D696631 00000000 6D696631 61766966 6D696166 000000F4 + return len(buf) > 1 && bytes.Equal(buf[:28], []byte{ + 0x0, 0x0, 0x0, 0x1c, + 0x66, 0x74, 0x79, 0x70, + 0x6d, 0x69, 0x66, 0x31, + 0x0, 0x0, 0x0, 0x0, + 0x6d, 0x69, 0x66, 0x31, + 0x61, 0x76, 0x69, 0x66, + 0x6d, 0x69, 0x61, 0x66, + }) } - func getFileContentType(buffer []byte) string { - // Use the net/http package's handy DectectContentType function. Always returns a valid - // content-type by returning "application/octet-stream" if no others seemed to match. - contentType := http.DetectContentType(buffer) - return contentType + var avifType = filetype.NewType("avif", "image/avif") + filetype.AddMatcher(avifType, avifMatcher) + kind, _ := filetype.Match(buffer) + return kind.MIME.Value } -func fileCount(dir string) int { - count := 0 +func fileCount(dir string) int64 { + var count int64 = 0 _ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -106,22 +114,23 @@ func cleanProxyCache(cacheImagePath string) { } } -func genWebpAbs(RawImagePath string, ExhaustPath string, ImgFilename string, reqURI string) (string, string) { +func genOptimizedAbs(rawImagePath string, exhaustPath string, imageName string, reqURI string) (string, string) { // get file mod time - STAT, err := os.Stat(RawImagePath) + STAT, err := os.Stat(rawImagePath) if err != nil { log.Error(err.Error()) return "", "" } ModifiedTime := STAT.ModTime().Unix() // webpFilename: abc.jpg.png -> abc.jpg.png.1582558990.webp - WebpFilename := fmt.Sprintf("%s.%d.webp", ImgFilename, ModifiedTime) - cwd, _ := os.Getwd() + webpFilename := fmt.Sprintf("%s.%d.webp", imageName, ModifiedTime) + avifFilename := fmt.Sprintf("%s.%d.avif", imageName, ModifiedTime) // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp // Custom Exhaust: /path/to/exhaust/web_path/web_to/tsuki.jpg.1582558990.webp - WebpAbsolutePath := path.Clean(path.Join(ExhaustPath, path.Dir(reqURI), WebpFilename)) - return cwd, WebpAbsolutePath + webpAbsolutePath := path.Clean(path.Join(exhaustPath, path.Dir(reqURI), webpFilename)) + avifAbsolutePath := path.Clean(path.Join(exhaustPath, path.Dir(reqURI), avifFilename)) + return avifAbsolutePath, webpAbsolutePath } func genEtag(ImgAbsPath string) string { @@ -136,64 +145,60 @@ func genEtag(ImgAbsPath string) string { return fmt.Sprintf(`W/"%d-%08X"`, len(data), crc) } -func getCompressionRate(RawImagePath string, webpAbsPath string) string { +func getCompressionRate(RawImagePath string, optimizedImg string) string { originFileInfo, err := os.Stat(RawImagePath) if err != nil { log.Warnf("fail to get raw image %v", err) return "" } - webpFileInfo, err := os.Stat(webpAbsPath) + optimizedFileInfo, err := os.Stat(optimizedImg) if err != nil { log.Warnf("fail to get webp image %v", err) return "" } - compressionRate := float64(webpFileInfo.Size()) / float64(originFileInfo.Size()) - log.Debugf("The compress rate is %d/%d=%.2f", originFileInfo.Size(), webpFileInfo.Size(), compressionRate) + compressionRate := float64(optimizedFileInfo.Size()) / float64(originFileInfo.Size()) + log.Debugf("The compress rate is %d/%d=%.2f", originFileInfo.Size(), optimizedFileInfo.Size(), compressionRate) return fmt.Sprintf(`%.2f`, compressionRate) } -func goOrigin(header, ua string) bool { - // We'll first check accept headers, if accept headers is false, we'll then go to UA part - if headerOrigin(header) && uaOrigin(ua) { - return true - } else { - return false - } -} +func guessSupportedFormat(header *fasthttp.RequestHeader) []string { + var supported = map[string]bool{ + "raw": true, + "webp": false, + "avif": false} -func uaOrigin(ua string) bool { - // iOS 14 and iPadOS 14 supports webp, the identification token is iPhone OS 14_2_1 and CPU OS 14_2 - // for more information, please check test case - if strings.Contains(ua, "iPhone OS 14") || strings.Contains(ua, "CPU OS 14") { - // this is iOS 14/iPadOS 14 - return false - } else if strings.Contains(ua, "Firefox") || strings.Contains(ua, "Chrome") { - // Chrome or firefox on macOS Windows + var ua = string(header.Peek("user-agent")) + var accept = strings.ToLower(string(header.Peek("accept"))) + log.Debugf("%s\t%s\n", ua, accept) + + if strings.Contains(accept, "image/webp") { + supported["webp"] = true + } + if strings.Contains(accept, "image/avif") { + supported["avif"] = true + } + + // chrome on iOS will not send valid image accept header + if strings.Contains(ua, "iPhone OS 14") || strings.Contains(ua, "CPU OS 14") || + strings.Contains(ua, "iPhone OS 15") || strings.Contains(ua, "CPU OS 15") { + supported["webp"] = true } else if strings.Contains(ua, "Android") || strings.Contains(ua, "Linux") { - // on Android and Linux - } else if strings.Contains(ua, "FxiOS") || strings.Contains(ua, "CriOS") { - //firefox and Chrome on iOS - return true - } else { - return true + supported["webp"] = true } - return false + + var accepted []string + for k, v := range supported { + if v { + accepted = append(accepted, k) + } + } + return accepted + } -func headerOrigin(header string) bool { - // Webkit is really weird especially on iOS, it doesn't even send out effective accept headers. - // Head to test case if you want to know more - if strings.Contains(header, "image/webp") { - return false - } else { - // go to origin - return true - } -} - -func chooseProxy(proxyRawSize string, webpAbsPath string) bool { +func chooseProxy(proxyRawSize string, optimizedAbs string) bool { var proxyRaw, _ = strconv.Atoi(proxyRawSize) - webp, _ := ioutil.ReadFile(webpAbsPath) + webp, _ := ioutil.ReadFile(optimizedAbs) if len(webp) > proxyRaw { return true } else { @@ -201,12 +206,16 @@ func chooseProxy(proxyRawSize string, webpAbsPath string) bool { } } -func chooseLocalSmallerFile(rawImageAbs, webpAbsPath string) string { - raw, _ := ioutil.ReadFile(rawImageAbs) - webp, _ := ioutil.ReadFile(webpAbsPath) - if len(webp) > len(raw) { - return rawImageAbs - } else { - return webpAbsPath +func findSmallestFiles(files []string) string { + // walk files + var small int64 + var final string + for _, f := range files { + stat, _ := os.Stat(f) + if stat.Size() < small || small == 0 { + small = stat.Size() + final = f + } } + return final } diff --git a/helper_test.go b/helper_test.go index 8af14f5..65a1ede 100644 --- a/helper_test.go +++ b/helper_test.go @@ -43,7 +43,7 @@ func TestImageExists(t *testing.T) { } func TestGenWebpAbs(t *testing.T) { - cwd, cooked := genWebpAbs("./pics/webp_server.png", "/tmp", + cwd, cooked := genOptimizedAbs("./pics/webp_server.png", "/tmp", "test", "a") if !strings.Contains(cwd, "webp_server_go") { t.Logf("Result: [%v], Expected: [%v]", cwd, "webp_server_go") diff --git a/prefetch.go b/prefetch.go index b55ab54..438d0f9 100644 --- a/prefetch.go +++ b/prefetch.go @@ -2,49 +2,49 @@ package main import ( "fmt" + "github.com/schollz/progressbar/v3" + log "github.com/sirupsen/logrus" "os" "path" "path/filepath" - "strconv" "strings" "time" - - log "github.com/sirupsen/logrus" ) -func prefetchImages(confImgPath string, ExhaustPath string, QUALITY string) { - var sTime = time.Now() +func prefetchImages(confImgPath string, ExhaustPath string) { // maximum ongoing prefetch is depending on your core of CPU + var sTime = time.Now() log.Infof("Prefetching using %d cores", jobs) var finishChan = make(chan int, jobs) for i := 0; i < jobs; i++ { - finishChan <- 0 + finishChan <- 1 } //prefetch, recursive through the dir all := fileCount(confImgPath) - count := 0 + var bar = progressbar.Default(all, "Prefetching...") + //count := 0 err := filepath.Walk(confImgPath, func(picAbsPath string, info os.FileInfo, err error) error { if err != nil { return err } + if info.IsDir() { + return nil + } // RawImagePath string, ImgFilename string, reqURI string proposedURI := strings.Replace(picAbsPath, confImgPath, "", 1) - _, p2 := genWebpAbs(picAbsPath, ExhaustPath, info.Name(), proposedURI) - q, _ := strconv.ParseFloat(QUALITY, 32) - _ = os.MkdirAll(path.Dir(p2), 0755) - go webpEncoder(picAbsPath, p2, float32(q)) - count += <-finishChan - //progress bar - _, _ = fmt.Fprintf(os.Stdout, "[Webp Server started] - convert in progress: %d/%d\r", count, all) + avif, webp := genOptimizedAbs(picAbsPath, ExhaustPath, info.Name(), proposedURI) + _ = os.MkdirAll(path.Dir(avif), 0755) + go convertFilter(picAbsPath, avif, webp, finishChan) + _ = bar.Add(<-finishChan) return nil }) + if err != nil { - log.Debug(err) + log.Errorln(err) } elapsed := time.Since(sTime) - _, _ = fmt.Fprintf(os.Stdout, "Prefetch completeY(^_^)Y\n\n") - _, _ = fmt.Fprintf(os.Stdout, "convert %d file in %s (^_^)Y\n\n", count, elapsed) + _, _ = fmt.Fprintf(os.Stdout, "Prefetch completeY(^_^)Y in %s\n\n", elapsed) } diff --git a/router.go b/router.go index 9aae749..501c760 100644 --- a/router.go +++ b/router.go @@ -5,12 +5,11 @@ import ( "fmt" "github.com/gofiber/fiber/v2" log "github.com/sirupsen/logrus" + "io/ioutil" "net/http" "net/url" "os" "path" - "path/filepath" - "strconv" "strings" ) @@ -24,14 +23,12 @@ func convert(c *fiber.Ctx) error { rawImageAbs = path.Join(config.ImgPath, reqURI) // /home/xxx/mypic/123.jpg } var imgFilename = path.Base(reqURI) // pure filename, 123.jpg - var finalFile string // We'll only need one c.sendFile() - var ua = c.Get("User-Agent") - var accept = c.Get("accept") - log.Debugf("Incoming connection from %s@%s with %s", ua, c.IP(), imgFilename) + log.Debugf("Incoming connection from %s %s", c.IP(), imgFilename) - needOrigin := goOrigin(accept, ua) - if needOrigin { - log.Debugf("A Safari/IE/whatever user has arrived...%s", ua) + goodFormat := guessSupportedFormat(&c.Request().Header) + + // old browser only + if len(goodFormat) == 1 { c.Set("ETag", genEtag(rawImageAbs)) if proxyMode { localRemoteTmpPath := remoteRaw + reqURI @@ -42,7 +39,6 @@ func convert(c *fiber.Ctx) error { } } - // check ext var allowed = false for _, ext := range config.AllowedTypes { haystack := strings.ToLower(imgFilename) @@ -81,45 +77,27 @@ func convert(c *fiber.Ctx) error { return errors.New(msg) } - _, webpAbsPath := genWebpAbs(rawImageAbs, config.ExhaustPath, imgFilename, reqURI) + // generate with timestamp to make sure files are update-to-date + avifAbs, webpAbs := genOptimizedAbs(rawImageAbs, config.ExhaustPath, imgFilename, reqURI) + convertFilter(rawImageAbs, avifAbs, webpAbs, nil) - if imageExists(webpAbsPath) { - finalFile = webpAbsPath - } else { - // we don't have abc.jpg.png1582558990.webp - // delete the old pic and convert a new one. - // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp - destHalfFile := path.Clean(path.Join(webpAbsPath, path.Dir(reqURI), imgFilename)) - matches, err := filepath.Glob(destHalfFile + "*") - if err != nil { - log.Error(err.Error()) - } else { - // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558100.webp <- older ones will be removed - // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp <- keep the latest one - for _, p := range matches { - if strings.Compare(destHalfFile, p) != 0 { - _ = os.Remove(p) - } - } + var availableFiles = []string{rawImageAbs} + for _, v := range goodFormat { + if "avif" == v { + availableFiles = append(availableFiles, avifAbs) } - - //for webp, we need to create dir first - err = os.MkdirAll(path.Dir(webpAbsPath), 0755) - q, _ := strconv.ParseFloat(config.Quality, 32) - err = webpEncoder(rawImageAbs, webpAbsPath, float32(q)) - - if err != nil { - log.Error(err) - _ = c.SendStatus(400) - _ = c.Send([]byte("Bad file. " + err.Error())) - return err + if "webp" == v { + availableFiles = append(availableFiles, webpAbs) } - finalFile = webpAbsPath } + + var finalFile string + finalFile = findSmallestFiles(availableFiles) etag := genEtag(finalFile) c.Set("ETag", etag) - c.Set("X-Compression-Rate", getCompressionRate(rawImageAbs, webpAbsPath)) - finalFile = chooseLocalSmallerFile(rawImageAbs, webpAbsPath) + c.Set("X-Compression-Rate", getCompressionRate(rawImageAbs, finalFile)) + buf, _ := ioutil.ReadFile(finalFile) + c.Set("content-type", getFileContentType(buf)) return c.SendFile(finalFile) } @@ -141,12 +119,8 @@ func proxyHandler(c *fiber.Ctx, reqURI string) error { cleanProxyCache(config.ExhaustPath + reqURI + "*") localRawImagePath := remoteRaw + reqURI _ = fetchRemoteImage(localRawImagePath, realRemoteAddr) - q, _ := strconv.ParseFloat(config.Quality, 32) _ = os.MkdirAll(path.Dir(localEtagWebPPath), 0755) - err := webpEncoder(localRawImagePath, localEtagWebPPath, float32(q)) - if err != nil { - log.Warning(err) - } + webpEncoder(localRawImagePath, localEtagWebPPath, config.Quality) chooseProxy(remoteLength, localEtagWebPPath) return c.SendFile(localEtagWebPPath) } diff --git a/webp-server.go b/webp-server.go index bbe9315..69de167 100644 --- a/webp-server.go +++ b/webp-server.go @@ -41,7 +41,7 @@ func deferInit() { FullTimestamp: true, TimestampFormat: "2006-01-02 15:04:05", CallerPrettyfier: func(f *runtime.Frame) (string, string) { - return fmt.Sprintf("[%s()]", f.Function), "" + return fmt.Sprintf("[%d:%s()]", f.Line, f.Function), "" }, } log.SetFormatter(Formatter) @@ -99,7 +99,7 @@ Develop by WebP Server team. https://github.com/webp-sh`, version) switchProxyMode() if prefetch { - go prefetchImages(config.ImgPath, config.ExhaustPath, config.Quality) + go prefetchImages(config.ImgPath, config.ExhaustPath) } app := fiber.New(fiber.Config{