diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..c72036e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.github/* +exhaust/* +pics/* +remote-raw/* +scripts/* +.git/* +README.md +LICENSE +Makefile \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 76c2524..3938344 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,6 +1,6 @@ version: 2 updates: - - package-ecosystem: "" + - package-ecosystem: "gomod" directory: "/" schedule: interval: "daily" diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml new file mode 100644 index 0000000..5d8ece7 --- /dev/null +++ b/.github/workflows/CI.yaml @@ -0,0 +1,21 @@ +name: CI +on: + push: + +jobs: + ci: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: true + + - name: Install Go + uses: actions/setup-go@v2 + with: + go-version: 1.17.3 + + - name: run test cases + run: sudo apt install libaom-dev && make test && make + diff --git a/Dockerfile b/Dockerfile index 95c75bf..df31bb7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,16 @@ -FROM golang:alpine as builder +FROM golang:1.17.4-alpine as builder ARG IMG_PATH=/opt/pics ARG EXHAUST_PATH=/opt/exhaust -RUN apk update && apk add alpine-sdk && mkdir /build +RUN apk update && apk add alpine-sdk aom-dev && mkdir /build COPY go.mod /build RUN cd /build && go mod download COPY . /build -RUN cd /build && sed -i "s|.\/pics|${IMG_PATH}|g" config.json \ -&& sed -i "s|\"\"|\"${EXHAUST_PATH}\"|g" config.json \ -&& sed -i 's/127.0.0.1/0.0.0.0/g' config.json \ -&& make docker +RUN cd /build && sed -i "s|.\/pics|${IMG_PATH}|g" config.json \ + && sed -i "s|\"\"|\"${EXHAUST_PATH}\"|g" config.json \ + && sed -i 's/127.0.0.1/0.0.0.0/g' config.json \ + && go build -ldflags="-s -w" -o webp-server . diff --git a/Makefile b/Makefile index 63d1c4e..2706e3e 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,7 @@ default: make clean go build -o builds/webp-server-$(OS)-$(ARCH) . ls builds + all: make clean ./scripts/build.sh $(OS) $(ARCH) @@ -26,4 +27,4 @@ clean: rm -rf prefetch docker: - go build -ldflags="-s -w" -o webp-server . \ No newline at end of file + DOCKER_BUILDKIT=1 docker build -t webpsh/webps . \ No newline at end of file diff --git a/config.go b/config.go index 706aa8d..15a5ca6 100644 --- a/config.go +++ b/config.go @@ -8,9 +8,10 @@ type Config struct { Host string `json:"HOST"` Port string `json:"PORT"` ImgPath string `json:"IMG_PATH"` - Quality string `json:"QUALITY"` + Quality float32 `json:"QUALITY,string"` AllowedTypes []string `json:"ALLOWED_TYPES"` ExhaustPath string `json:"EXHAUST_PATH"` + EnableAVIF bool `json:"ENABLE_AVIF"` } var ( @@ -21,7 +22,7 @@ var ( prefetch, proxyMode bool remoteRaw = "remote-raw" config Config - version = "0.3.2" + version = "0.4.0" releaseUrl = "https://github.com/webp-sh/webp_server_go/releases/latest/download/" ) @@ -33,7 +34,8 @@ const ( "QUALITY": "80", "IMG_PATH": "./pics", "EXHAUST_PATH": "./exhaust", - "ALLOWED_TYPES": ["jpg","png","jpeg","bmp"] + "ALLOWED_TYPES": ["jpg","png","jpeg","bmp"], + "ENABLE_AVIF": false }` sampleSystemd = ` @@ -53,3 +55,8 @@ RestartSec=3s [Install] WantedBy=multi-user.target` ) + +const ( + webpMax = 16383 + avifMax = 65536 +) diff --git a/encoder.go b/encoder.go index bf50b95..cebc20a 100644 --- a/encoder.go +++ b/encoder.go @@ -3,70 +3,152 @@ package main import ( "bytes" "errors" + "fmt" + "github.com/Kagami/go-avif" + "github.com/chai2010/webp" + log "github.com/sirupsen/logrus" + "golang.org/x/image/bmp" "image" - "image/gif" "image/jpeg" "image/png" "io/ioutil" + "os" "path" + "path/filepath" + "runtime" "strings" - - log "github.com/sirupsen/logrus" - - "github.com/chai2010/webp" - "golang.org/x/image/bmp" ) -func webpEncoder(p1, p2 string, quality float32, Log bool, c chan int) (err error) { - // if convert fails, return error; success nil +func convertFilter(raw, avifPath, webpPath string, c chan int) { + // all absolute paths - log.Debugf("target: %s with quality of %f", path.Base(p1), quality) - var buf bytes.Buffer - var img image.Image + if !imageExists(avifPath) && config.EnableAVIF { + convertImage(raw, avifPath, "avif") + } - data, err := ioutil.ReadFile(p1) + if !imageExists(webpPath) { + convertImage(raw, webpPath, "webp") + } + + if c != nil { + c <- 1 + } +} + +func convertImage(raw, optimized, itype string) { + // we don't have abc.jpg.png1582558990.webp + // delete the old pic and convert a new one. + // optimized: /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp + // we'll delete file starts with /home/webp_server/exhaust/path/to/tsuki.jpg.ts.itype + + s := strings.Split(path.Base(optimized), ".") + pattern := path.Join(path.Dir(optimized), s[0]+"."+s[1]+".*."+s[len(s)-1]) + + matches, err := filepath.Glob(pattern) if err != nil { - chanErr(c) - return + log.Error(err.Error()) + } else { + for _, p := range matches { + _ = os.Remove(p) + } + } + + //we need to create dir first + err = os.MkdirAll(path.Dir(optimized), 0755) + //q, _ := strconv.ParseFloat(config.Quality, 32) + + switch itype { + case "webp": + webpEncoder(raw, optimized, config.Quality) + case "avif": + avifEncoder(raw, optimized, config.Quality) + } + +} + +func readRawImage(imgPath string, maxPixel int) (img image.Image, err error) { + data, err := ioutil.ReadFile(imgPath) + if err != nil { + log.Errorln(err) } contentType := getFileContentType(data[:512]) if strings.Contains(contentType, "jpeg") { - img, _ = jpeg.Decode(bytes.NewReader(data)) + img, err = jpeg.Decode(bytes.NewReader(data)) } else if strings.Contains(contentType, "png") { - img, _ = png.Decode(bytes.NewReader(data)) + img, err = png.Decode(bytes.NewReader(data)) } else if strings.Contains(contentType, "bmp") { - img, _ = bmp.Decode(bytes.NewReader(data)) - } else if strings.Contains(contentType, "gif") { - // TODO: need to support animated webp - log.Warn("Gif support is not perfect!") - img, _ = gif.Decode(bytes.NewReader(data)) + img, err = bmp.Decode(bytes.NewReader(data)) + } + if err != nil || img == nil { + errinfo := fmt.Sprintf("image file %s is corrupted: %v", imgPath, err) + log.Errorln(errinfo) + return nil, errors.New(errinfo) } - if img == nil { - msg := "image file " + path.Base(p1) + " is corrupted or not supported" - log.Debug(msg) - err = errors.New(msg) - chanErr(c) - return + x, y := img.Bounds().Max.X, img.Bounds().Max.Y + if x > maxPixel || y > maxPixel { + errinfo := fmt.Sprintf("WebP: %s(%dx%d) is too large", imgPath, x, y) + log.Warnf(errinfo) + return nil, errors.New(errinfo) } - if err = webp.Encode(&buf, img, &webp.Options{Lossless: false, Quality: quality}); err != nil { - log.Error(err) - chanErr(c) - return - } - if err = ioutil.WriteFile(p2, buf.Bytes(), 0644); err != nil { - log.Error(err) - chanErr(c) - return - } - - if Log { - log.Info("Save to " + p2 + " ok!\n") - } - - chanErr(c) - - return nil + return img, nil +} + +func avifEncoder(p1, p2 string, quality float32) { + var img image.Image + dst, err := os.Create(p2) + if err != nil { + log.Fatalf("Can't create destination file: %v", err) + } + // AVIF has a maximum resolution of 65536 x 65536 pixels. + img, err = readRawImage(p1, avifMax) + if err != nil { + return + } + + err = avif.Encode(dst, img, &avif.Options{ + Threads: runtime.NumCPU(), + Speed: avif.MaxSpeed, + Quality: int((100 - quality) / 100 * avif.MaxQuality), + SubsampleRatio: nil, + }) + + if err != nil { + log.Warnf("Can't encode source image: %v to AVIF", err) + } + + convertLog("AVIF", p1, p2, quality) +} + +func webpEncoder(p1, p2 string, quality float32) { + // if convert fails, return error; success nil + var buf bytes.Buffer + var img image.Image + // The maximum pixel dimensions of a WebP image is 16383 x 16383. + img, err := readRawImage(p1, webpMax) + if err != nil { + return + } + + err = webp.Encode(&buf, img, &webp.Options{Lossless: false, Quality: quality}) + if err != nil { + log.Warnf("Can't encode source image: %v to WebP", err) + } + + if err := ioutil.WriteFile(p2, buf.Bytes(), 0644); err != nil { + log.Error(err) + return + } + + convertLog("WebP", p1, p2, quality) + +} + +func convertLog(itype, p1 string, p2 string, quality float32) { + oldf, _ := os.Stat(p1) + newf, _ := os.Stat(p2) + log.Infof("%s@%.2f%%: %s->%s %d->%d %.2f%% deflated", itype, quality, + p1, p2, oldf.Size(), newf.Size(), float32(newf.Size())/float32(oldf.Size())*100) } diff --git a/encoder_test.go b/encoder_test.go index 693baf2..cfec0cd 100644 --- a/encoder_test.go +++ b/encoder_test.go @@ -4,65 +4,63 @@ import ( "github.com/stretchr/testify/assert" "io/ioutil" "os" + "path" "path/filepath" + "strings" "testing" ) func walker() []string { var list []string - _ = filepath.Walk("./pics", func(path string, info os.FileInfo, err error) error { - if !info.IsDir() { - list = append(list, path) + _ = filepath.Walk("./pics", func(p string, info os.FileInfo, err error) error { + if !info.IsDir() && !strings.HasPrefix(path.Base(p), ".") { + list = append(list, p) } return nil }) return list } -func TestWebpEncoder(t *testing.T) { - var webp = "/tmp/test-result.webp" +func TestWebPEncoder(t *testing.T) { + // Go through every files + var dest = "/tmp/test-result" var target = walker() - for _, f := range target { - //fmt.Println(b, c, webp) - runEncoder(t, f, webp) + runEncoder(t, f, dest) } - _ = os.Remove(webp) + _ = os.Remove(dest) +} - // test error - err := webpEncoder("./pics/empty.jpg", webp, 80, true, nil) - assert.NotNil(t, err) - _ = os.Remove(webp) +func TestAvifEncoder(t *testing.T) { + // Only one file: img_over_16383px.jpg might cause memory issues on CI environment + var dest = "/tmp/test-result" + avifEncoder("./pics/big.jpg", dest, 80) + assertType(t, dest, "image/avif") } func TestNonExistImage(t *testing.T) { - var webp = "/tmp/test-result.webp" - // test error - var err = webpEncoder("./pics/empty.jpg", webp, 80, true, nil) - assert.NotNil(t, err) - _ = os.Remove(webp) + var dest = "/tmp/test-result" + webpEncoder("./pics/empty.jpg", dest, 80) + avifEncoder("./pics/empty.jpg", dest, 80) } func TestConvertFail(t *testing.T) { - var webp = "/tmp/test-result.webp" - var err = webpEncoder("./pics/webp_server.jpg", webp, -1, true, nil) - assert.NotNil(t, t, err) + var dest = "/tmp/test-result" + webpEncoder("./pics/webp_server.jpg", dest, -1) + avifEncoder("./pics/webp_server.jpg", dest, -1) } -func runEncoder(t *testing.T, file string, webp string) { - var c chan int - //t.Logf("convert from %s to %s", file, webp) - var err = webpEncoder(file, webp, 80, true, c) - if file == "pics/empty.jpg" && err != nil { +func runEncoder(t *testing.T, file string, dest string) { + if file == "pics/empty.jpg" { t.Log("Empty file, that's okay.") - } else if err != nil { - t.Fatalf("Fatal, convert failed for %s: %v ", file, err) - } - - data, _ := ioutil.ReadFile(webp) - types := getFileContentType(data[:512]) - if types != "image/webp" { - t.Fatal("Fatal, file type is wrong!") } + webpEncoder(file, dest, 80) + assertType(t, dest, "image/webp") } + +func assertType(t *testing.T, dest, mime string) { + data, _ := ioutil.ReadFile(dest) + types := getFileContentType(data[:512]) + assert.Equalf(t, mime, types, "File %s should be %s", dest, mime) +} diff --git a/go.mod b/go.mod index 156ad42..37130d9 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,18 @@ module webp_server_go go 1.15 require ( + github.com/Kagami/go-avif v0.1.0 github.com/chai2010/webp v1.1.0 github.com/gofiber/fiber/v2 v2.4.0 - github.com/sirupsen/logrus v1.6.0 + github.com/h2non/filetype v1.1.3 + github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect + github.com/schollz/progressbar/v3 v3.8.3 + github.com/sirupsen/logrus v1.8.1 + github.com/staktrace/go-update v0.0.0-20210525161054-fc019945f9a2 github.com/stretchr/testify v1.3.0 + github.com/valyala/fasthttp v1.18.0 golang.org/x/image v0.0.0-20200119044424-58c23975cae1 + golang.org/x/sys v0.0.0-20211204120058-94396e421777 // indirect ) replace ( diff --git a/go.sum b/go.sum index 4f862f3..39dd9e1 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,37 @@ +github.com/Kagami/go-avif v0.1.0 h1:8GHAGLxCdFfhpd4Zg8j1EqO7rtcQNenxIDerC/uu68w= +github.com/Kagami/go-avif v0.1.0/go.mod h1:OPmPqzNdQq3+sXm0HqaUJQ9W/4k+Elbc3RSfJUemDKA= github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= +github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= +github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/klauspost/compress v1.10.7 h1:7rix8v8GpI3ZBb0nSozFRgbtXKv+hOe+qfEpZqybrAg= github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/schollz/progressbar/v3 v3.8.3 h1:FnLGl3ewlDUP+YdSwveXBaXs053Mem/du+wr7XSYKl8= +github.com/schollz/progressbar/v3 v3.8.3/go.mod h1:pWnVCjSBZsT2X3nx9HfRdnCDrpbevliMeoEVhStwHko= github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/staktrace/go-update v0.0.0-20210525161054-fc019945f9a2 h1:kyTDvRL8TyTHOx0aK1PKMPVfkI7rCLDC1nrKF7SYOBc= +github.com/staktrace/go-update v0.0.0-20210525161054-fc019945f9a2/go.mod h1:yzHsbjWRHVF1vVGsluwqCc2TvogFSx9C8TeBh34cMb0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= @@ -23,16 +45,29 @@ github.com/webp-sh/webp v1.2.0 h1:WiAR1M4Cz50Ehv0vP8VaBZwuTqBgLxcGMaBV23zLVDA= github.com/webp-sh/webp v1.2.0/go.mod h1:DWmBXPtpA/zfTgEgWxAlsER3B7nXFvDtCi1YY+K5N9w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/image v0.0.0-20200119044424-58c23975cae1 h1:5h3ngYt7+vXCDZCup/HkCQgW5XwmSvR/nA2JmJ0RErg= golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201210223839-7e3030f88018 h1:XKi8B/gRBuTZN1vU9gFsLMm6zVz5FSCDzm8JYACnjy8= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201210223839-7e3030f88018/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0 h1:xrCZDmdtoloIiooiA9q0OQb9r8HejIHYoHGhGCe1pGg= +golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211204120058-94396e421777 h1:QAkhGVjOxMa+n4mlsAWeAU+BMZmimQAaNiMu+iUi94E= +golang.org/x/sys v0.0.0-20211204120058-94396e421777/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/helper.go b/helper.go index 7a9eb5f..d9e3130 100644 --- a/helper.go +++ b/helper.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "fmt" "hash/crc32" "io" @@ -11,26 +12,35 @@ import ( "path/filepath" "strconv" + "github.com/valyala/fasthttp" + "strings" + "github.com/h2non/filetype" log "github.com/sirupsen/logrus" ) -func chanErr(ccc chan int) { - if ccc != nil { - ccc <- 1 - } +func avifMatcher(buf []byte) bool { + // 0000001C 66747970 6D696631 00000000 6D696631 61766966 6D696166 000000F4 + return len(buf) > 1 && bytes.Equal(buf[:28], []byte{ + 0x0, 0x0, 0x0, 0x1c, + 0x66, 0x74, 0x79, 0x70, + 0x6d, 0x69, 0x66, 0x31, + 0x0, 0x0, 0x0, 0x0, + 0x6d, 0x69, 0x66, 0x31, + 0x61, 0x76, 0x69, 0x66, + 0x6d, 0x69, 0x61, 0x66, + }) } - func getFileContentType(buffer []byte) string { - // Use the net/http package's handy DectectContentType function. Always returns a valid - // content-type by returning "application/octet-stream" if no others seemed to match. - contentType := http.DetectContentType(buffer) - return contentType + var avifType = filetype.NewType("avif", "image/avif") + filetype.AddMatcher(avifType, avifMatcher) + kind, _ := filetype.Match(buffer) + return kind.MIME.Value } -func fileCount(dir string) int { - count := 0 +func fileCount(dir string) int64 { + var count int64 = 0 _ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -53,6 +63,17 @@ func imageExists(filename string) bool { return !info.IsDir() } +func checkAllowedType(imgFilename string) bool { + imgFilename = strings.ToLower(imgFilename) + for _, allowedType := range config.AllowedTypes { + allowedType = "." + strings.ToLower(allowedType) + if strings.HasSuffix(imgFilename, allowedType) { + return true + } + } + return false +} + // Check for remote filepath, e.g: https://test.webp.sh/node.png // return StatusCode, etagValue and length func getRemoteImageInfo(fileUrl string) (int, string, string) { @@ -106,22 +127,24 @@ func cleanProxyCache(cacheImagePath string) { } } -func genWebpAbs(RawImagePath string, ExhaustPath string, ImgFilename string, reqURI string) (string, string) { +func genOptimizedAbs(rawImagePath string, exhaustPath string, imageName string, reqURI string) (string, string) { // get file mod time - STAT, err := os.Stat(RawImagePath) + STAT, err := os.Stat(rawImagePath) if err != nil { log.Error(err.Error()) return "", "" } ModifiedTime := STAT.ModTime().Unix() // webpFilename: abc.jpg.png -> abc.jpg.png.1582558990.webp - WebpFilename := fmt.Sprintf("%s.%d.webp", ImgFilename, ModifiedTime) - cwd, _ := os.Getwd() + webpFilename := fmt.Sprintf("%s.%d.webp", imageName, ModifiedTime) + // avifFilename: abc.jpg.png -> abc.jpg.png.1582558990.avif + avifFilename := fmt.Sprintf("%s.%d.avif", imageName, ModifiedTime) // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp // Custom Exhaust: /path/to/exhaust/web_path/web_to/tsuki.jpg.1582558990.webp - WebpAbsolutePath := path.Clean(path.Join(ExhaustPath, path.Dir(reqURI), WebpFilename)) - return cwd, WebpAbsolutePath + webpAbsolutePath := path.Clean(path.Join(exhaustPath, path.Dir(reqURI), webpFilename)) + avifAbsolutePath := path.Clean(path.Join(exhaustPath, path.Dir(reqURI), avifFilename)) + return avifAbsolutePath, webpAbsolutePath } func genEtag(ImgAbsPath string) string { @@ -136,64 +159,60 @@ func genEtag(ImgAbsPath string) string { return fmt.Sprintf(`W/"%d-%08X"`, len(data), crc) } -func getCompressionRate(RawImagePath string, webpAbsPath string) string { +func getCompressionRate(RawImagePath string, optimizedImg string) string { originFileInfo, err := os.Stat(RawImagePath) if err != nil { - log.Warnf("fail to get raw image %v", err) + log.Warnf("Failed to get raw image %v", err) return "" } - webpFileInfo, err := os.Stat(webpAbsPath) + optimizedFileInfo, err := os.Stat(optimizedImg) if err != nil { - log.Warnf("fail to get webp image %v", err) + log.Warnf("Failed to get optimized image %v", err) return "" } - compressionRate := float64(webpFileInfo.Size()) / float64(originFileInfo.Size()) - log.Debugf("The compress rate is %d/%d=%.2f", originFileInfo.Size(), webpFileInfo.Size(), compressionRate) + compressionRate := float64(optimizedFileInfo.Size()) / float64(originFileInfo.Size()) + log.Debugf("The compression rate is %d/%d=%.2f", originFileInfo.Size(), optimizedFileInfo.Size(), compressionRate) return fmt.Sprintf(`%.2f`, compressionRate) } -func goOrigin(header, ua string) bool { - // We'll first check accept headers, if accept headers is false, we'll then go to UA part - if headerOrigin(header) && uaOrigin(ua) { - return true - } else { - return false - } -} +func guessSupportedFormat(header *fasthttp.RequestHeader) []string { + var supported = map[string]bool{ + "raw": true, + "webp": false, + "avif": false} -func uaOrigin(ua string) bool { - // iOS 14 and iPadOS 14 supports webp, the identification token is iPhone OS 14_2_1 and CPU OS 14_2 - // for more information, please check test case - if strings.Contains(ua, "iPhone OS 14") || strings.Contains(ua, "CPU OS 14") { - // this is iOS 14/iPadOS 14 - return false - } else if strings.Contains(ua, "Firefox") || strings.Contains(ua, "Chrome") { - // Chrome or firefox on macOS Windows + var ua = string(header.Peek("user-agent")) + var accept = strings.ToLower(string(header.Peek("accept"))) + log.Debugf("%s\t%s\n", ua, accept) + + if strings.Contains(accept, "image/webp") { + supported["webp"] = true + } + if strings.Contains(accept, "image/avif") { + supported["avif"] = true + } + + // chrome on iOS will not send valid image accept header + if strings.Contains(ua, "iPhone OS 14") || strings.Contains(ua, "CPU OS 14") || + strings.Contains(ua, "iPhone OS 15") || strings.Contains(ua, "CPU OS 15") { + supported["webp"] = true } else if strings.Contains(ua, "Android") || strings.Contains(ua, "Linux") { - // on Android and Linux - } else if strings.Contains(ua, "FxiOS") || strings.Contains(ua, "CriOS") { - //firefox and Chrome on iOS - return true - } else { - return true + supported["webp"] = true } - return false + + var accepted []string + for k, v := range supported { + if v { + accepted = append(accepted, k) + } + } + return accepted + } -func headerOrigin(header string) bool { - // Webkit is really weird especially on iOS, it doesn't even send out effective accept headers. - // Head to test case if you want to know more - if strings.Contains(header, "image/webp") { - return false - } else { - // go to origin - return true - } -} - -func chooseProxy(proxyRawSize string, webpAbsPath string) bool { +func chooseProxy(proxyRawSize string, optimizedAbs string) bool { var proxyRaw, _ = strconv.Atoi(proxyRawSize) - webp, _ := ioutil.ReadFile(webpAbsPath) + webp, _ := ioutil.ReadFile(optimizedAbs) if len(webp) > proxyRaw { return true } else { @@ -201,12 +220,20 @@ func chooseProxy(proxyRawSize string, webpAbsPath string) bool { } } -func chooseLocalSmallerFile(rawImageAbs, webpAbsPath string) string { - raw, _ := ioutil.ReadFile(rawImageAbs) - webp, _ := ioutil.ReadFile(webpAbsPath) - if len(webp) > len(raw) { - return rawImageAbs - } else { - return webpAbsPath +func findSmallestFiles(files []string) string { + // walk files + var small int64 + var final string + for _, f := range files { + stat, err := os.Stat(f) + if err != nil { + log.Warnf("%s not found on filesystem", f) + continue + } + if stat.Size() < small || small == 0 { + small = stat.Size() + final = f + } } + return final } diff --git a/helper_test.go b/helper_test.go index 8af14f5..9217e8a 100644 --- a/helper_test.go +++ b/helper_test.go @@ -1,31 +1,33 @@ package main import ( + log "github.com/sirupsen/logrus" + "github.com/valyala/fasthttp" "io/ioutil" "net/http" "path/filepath" + "sort" "strings" "testing" "github.com/stretchr/testify/assert" ) -// test this file: go test -v -cover . +// test all files: go test -v -cover . +// test one case: go test -v -run TestSelectFormat + func TestGetFileContentType(t *testing.T) { - var data = []byte("hello") - var expected = "text/plain; charset=utf-8" + var data = []byte("remember remember the 5th of november") + var expected = "" var result = getFileContentType(data) - assert.Equalf(t, result, expected, "Result: [%s], Expected: [%s]", result, expected) - } func TestFileCount(t *testing.T) { - var data = ".github" - var expected = 2 + var data = "scripts" + var expected int64 = 2 var result = fileCount(data) assert.Equalf(t, result, expected, "Result: [%d], Expected: [%d]", result, expected) - } func TestImageExists(t *testing.T) { @@ -43,7 +45,7 @@ func TestImageExists(t *testing.T) { } func TestGenWebpAbs(t *testing.T) { - cwd, cooked := genWebpAbs("./pics/webp_server.png", "/tmp", + cwd, cooked := genOptimizedAbs("./pics/webp_server.png", "/tmp", "test", "a") if !strings.Contains(cwd, "webp_server_go") { t.Logf("Result: [%v], Expected: [%v]", cwd, "webp_server_go") @@ -71,108 +73,63 @@ func TestGenEtag(t *testing.T) { } -func TestGoOrigin(t *testing.T) { +func TestSelectFormat(t *testing.T) { // this is a complete test case for webp compatibility // func goOrigin(header, ua string) bool // UNLESS YOU KNOW WHAT YOU ARE DOING, DO NOT CHANGE THE TEST CASE MAPPING HERE. - var testCase = map[[2]string]bool{ + var fullSupport = []string{"avif", "webp", "raw"} + var webpSupport = []string{"webp", "raw"} + var jpegSupport = []string{"raw"} + var testCase = map[[2]string][]string{ + // Latest Chrome on Windows, macOS, linux, Android and iOS 13 + [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36"}: fullSupport, + [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36"}: fullSupport, + [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36"}: fullSupport, + [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", "Mozilla/5.0 (Linux; Android 10) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.60 Mobile Safari/537.36"}: fullSupport, + [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", "Mozilla/5.0 (Linux; Android 6.0; HTC M8t) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.74 Mobile Safari/537.36"}: fullSupport, + [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", "HTCM8t_LTE/1.0 Android/4.4 release/2013 Browser/WAP2.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 AppleWebKit/534.30"}: webpSupport, + [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 13_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/83.0.4103.63 Mobile/15E148 Safari/604.1"}: jpegSupport, + // macOS Catalina Safari - [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.1 Safari/605.1.15"}: true, - // macOS Chrome - [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.88 Safari/537.36"}: false, - // iOS14 Safari - [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 14_2_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.1 Mobile/15E148 Safari/604.1"}: false, - // iOS14 Chrome - [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 14_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/87.0.4280.77 Mobile/15E148 Safari/604.1"}: false, - // iPadOS 14 Safari - [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPad; CPU OS 14_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.1 Mobile/15E148 Safari/604.1"}: false, - // iPadOS 14 Chrome - [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPad; CPU OS 14_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/87.0.4280.77 Mobile/15E148 Safari/604.1"}: false, - // Warning: these three are real capture headers - I don't have iOS/iPadOS prior to version 14 - [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1 Safari/605.1.15"}: true, - [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPad; CPU OS 10_15_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/25.0 Mobile/15E148 Safari/605.1.15"}: true, - [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 13_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/83.0.4103.63 Mobile/15E148 Safari/604.1"}: true, - } + [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.1 Safari/605.1.15"}: jpegSupport, - for value, is := range testCase { - assert.Equalf(t, is, goOrigin(value[0], value[1]), "[%v]:[%s]", value, is) - } -} + // iOS14 Safari and Chrome + [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 14_2_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.1 Mobile/15E148 Safari/604.1"}: webpSupport, + [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 14_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/87.0.4280.77 Mobile/15E148 Safari/604.1"}: webpSupport, -func TestUaOrigin(t *testing.T) { - // reference: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent/Firefox - // https://developer.chrome.com/multidevice/user-agent#chrome_for_ios_user_agent + // iPadOS 14 Safari and Chrome + [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPad; CPU OS 14_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.1 Mobile/15E148 Safari/604.1"}: webpSupport, + [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPad; CPU OS 14_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/87.0.4280.77 Mobile/15E148 Safari/604.1"}: webpSupport, - // UNLESS YOU KNOW WHAT YOU ARE DOING, DO NOT CHANGE THE TEST CASE MAPPING HERE. - var testCase = map[string]bool{ - // Chrome on Windows, macOS, linux, iOS and Android - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36": false, - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36": false, - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36": false, - "Mozilla/5.0 (iPhone; CPU iPhone OS 13_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/83.0.4103.63 Mobile/15E148 Safari/604.1": true, - "Mozilla/5.0 (Linux; Android 10) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.60 Mobile Safari/537.36": false, - - // Firefox on Windows, macOS, linux, iOS and Android - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:76.0) Gecko/20100101 Firefox/76.0": false, - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:76.0) Gecko/20100101 Firefox/76.0": false, - "Mozilla/5.0 (X11; Linux i686; rv:76.0) Gecko/20100101 Firefox/76.0": false, - "Mozilla/5.0 (iPad; CPU OS 10_15_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/25.0 Mobile/15E148 Safari/605.1.15": true, - "Mozilla/5.0 (Android 10; Mobile; rv:68.0) Gecko/68.0 Firefox/68.0": false, - - // Safari on macOS and iOS - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1 Safari/605.1.15": true, - "Mozilla/5.0 (iPad; CPU OS 13_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Mobile/15E148 Safari/604.1": true, - - // WeChat on iOS, Windows, and Android - "Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_3 like Mac OS X) AppleWebKit/603.3.8 (KHTML, like Gecko) Mobile/14G60 wxwork/2.1.5 MicroMessenger/6.3.22": true, - "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 MicroMessenger/6.5.2.501 NetType/WIFI WindowsWechat QBCore/3.43.691.400 QQBrowser/9.0.2524.400": false, - "Mozilla/5.0 (Linux; Android 7.0; LG-H831 Build/NRD90U; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/68.0.3440.91 Mobile Safari/537.36 MicroMessenger/6.6.7.1303(0x26060743) NetType/WIFI Language/zh_TW": false, + // iOS 15 Safari, Firefox and Chrome + [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 15_1_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Mobile/15E148 Safari/604.1"}: webpSupport, + [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 15_1_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/39.0 Mobile/15E148 Safari/605.1.15"}: webpSupport, + [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 15_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/96.0.4664.53 Mobile/15E148 Safari/604.1"}: webpSupport, // IE - "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko": true, - "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)": true, - + [2]string{"application/x-ms-application, image/jpeg, application/xaml+xml, image/gif, image/pjpeg, application/x-ms-xbap, */*", "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko"}: jpegSupport, // Others - "PostmanRuntime/7.26.1": true, - "curl/7.64.1": true, + [2]string{"", "PostmanRuntime/7.26.1"}: jpegSupport, + [2]string{"", "curl/7.64.1"}: jpegSupport, + [2]string{"image/webp", "curl/7.64.1"}: webpSupport, + [2]string{"image/avif,image/webp", "curl/7.64.1"}: fullSupport, - // these four are captured from iOS14/iPadOS14, which supports webp - "Mozilla/5.0 (iPhone; CPU iPhone OS 14_2_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.1 Mobile/15E148 Safari/604.1": false, - "Mozilla/5.0 (iPhone; CPU iPhone OS 14_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/87.0.4280.77 Mobile/15E148 Safari/604.1\n": false, - "Mozilla/5.0 (iPad; CPU OS 14_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.1 Mobile/15E148 Safari/604.1": false, - "Mozilla/5.0 (iPad; CPU OS 14_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/87.0.4280.77 Mobile/15E148 Safari/604.1": false, + // some weird browsers + [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 15_1_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 MicroMessenger/8.0.16(0x18001033) NetType/WIFI Language/zh_CN"}: webpSupport, + [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", "Mozilla/5.0 (Linux; Android 6.0; HTC M8t Build/MRA58K; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/45.0.2454.95 Mobile Safari/537.36 MMWEBID/4285 MicroMessenger/8.0.15.2001(0x28000F41) Process/tools WeChat/arm32 Weixin GPVersion/1 NetType/WIFI Language/zh_CN ABI/arm32"}: webpSupport, + } + for browser, expected := range testCase { + var header fasthttp.RequestHeader + header.Set("accept", browser[0]) + header.Set("user-agent", browser[1]) + guessed := guessSupportedFormat(&header) + + sort.Strings(expected) + sort.Strings(guessed) + log.Infof("%s expected%s --- actual %s", browser, expected, guessed) + assert.Equal(t, expected, guessed) } - for browser, is := range testCase { - assert.Equalf(t, is, uaOrigin(browser), "[%v]:[%s]", is, browser) - } - -} - -func TestHeaderOrigin(t *testing.T) { - // Chrome: Accept: image/avif,image/webp,image/apng,image/*,*/*;q=0.8 - // Safari 版本14.0.1 (15610.2.11.51.10, 15610): Accept: image/png,image/svg+xml,image/*;q=0.8,video/*;q=0.8,*/*;q=0.5 - // Safari Technology Preview Release 116 (Safari 14.1, WebKit 15611.1.5.3) Accept: image/png,image/svg+xml,image/*;q=0.8,video/*;q=0.8,*/*;q=0.5 - - // UNLESS YOU KNOW WHAT YOU ARE DOING, DO NOT CHANGE THE TEST CASE MAPPING HERE. - var testCase = map[string]bool{ - "image/avif,image/webp,image/apng,image/*,*/*;q=0.8": false, - "*/*": true, - "image/png,image/svg+xml,image/*;q=0.8,video/*;q=0.8,*/*;q=0.5": true, - "I don't know what it is:-)": true, - } - for header, is := range testCase { - assert.Equalf(t, is, headerOrigin(header), "[%v]:[%s]", is, header) - } -} - -func TestChanErr(t *testing.T) { - var value = 2 - var testC = make(chan int, 2) - testC <- value - chanErr(testC) - value = <-testC - assert.Equal(t, 2, value) } func TestGetRemoteImageInfo(t *testing.T) { @@ -197,7 +154,7 @@ func TestFetchRemoteImage(t *testing.T) { err := fetchRemoteImage(fp, url) assert.Equal(t, err, nil) data, _ := ioutil.ReadFile(fp) - assert.Equal(t, "image/x-icon", getFileContentType(data)) + assert.Equal(t, "image/vnd.microsoft.icon", getFileContentType(data)) // test can't create file err = fetchRemoteImage("/", url) diff --git a/pics/big.webp b/pics/big.webp new file mode 100644 index 0000000..06383af Binary files /dev/null and b/pics/big.webp differ diff --git a/pics/img_over_16383px.jpg b/pics/img_over_16383px.jpg new file mode 100644 index 0000000..da76d8d Binary files /dev/null and b/pics/img_over_16383px.jpg differ diff --git a/pics/jpg_without_eoi.jpg b/pics/jpg_without_eoi.jpg new file mode 100644 index 0000000..934ec51 Binary files /dev/null and b/pics/jpg_without_eoi.jpg differ diff --git a/prefetch.go b/prefetch.go index 89e828f..1dbca1d 100644 --- a/prefetch.go +++ b/prefetch.go @@ -2,49 +2,49 @@ package main import ( "fmt" + "github.com/schollz/progressbar/v3" + log "github.com/sirupsen/logrus" "os" "path" "path/filepath" - "strconv" "strings" "time" - - log "github.com/sirupsen/logrus" ) -func prefetchImages(confImgPath string, ExhaustPath string, QUALITY string) { - var sTime = time.Now() +func prefetchImages(confImgPath string, ExhaustPath string) { // maximum ongoing prefetch is depending on your core of CPU + var sTime = time.Now() log.Infof("Prefetching using %d cores", jobs) var finishChan = make(chan int, jobs) for i := 0; i < jobs; i++ { - finishChan <- 0 + finishChan <- 1 } //prefetch, recursive through the dir all := fileCount(confImgPath) - count := 0 + var bar = progressbar.Default(all, "Prefetching...") err := filepath.Walk(confImgPath, func(picAbsPath string, info os.FileInfo, err error) error { if err != nil { return err } + if info.IsDir() { + return nil + } // RawImagePath string, ImgFilename string, reqURI string proposedURI := strings.Replace(picAbsPath, confImgPath, "", 1) - _, p2 := genWebpAbs(picAbsPath, ExhaustPath, info.Name(), proposedURI) - q, _ := strconv.ParseFloat(QUALITY, 32) - _ = os.MkdirAll(path.Dir(p2), 0755) - go webpEncoder(picAbsPath, p2, float32(q), false, finishChan) - count += <-finishChan - //progress bar - _, _ = fmt.Fprintf(os.Stdout, "[Webp Server started] - convert in progress: %d/%d\r", count, all) + avif, webp := genOptimizedAbs(picAbsPath, ExhaustPath, info.Name(), proposedURI) + _ = os.MkdirAll(path.Dir(avif), 0755) + log.Infof("Prefetching %s", picAbsPath) + go convertFilter(picAbsPath, avif, webp, finishChan) + _ = bar.Add(<-finishChan) return nil }) + if err != nil { - log.Debug(err) + log.Errorln(err) } elapsed := time.Since(sTime) - _, _ = fmt.Fprintf(os.Stdout, "Prefetch completeY(^_^)Y\n\n") - _, _ = fmt.Fprintf(os.Stdout, "convert %d file in %s (^_^)Y\n\n", count, elapsed) + _, _ = fmt.Fprintf(os.Stdout, "Prefetch completeY(^_^)Y in %s\n\n", elapsed) } diff --git a/prefetch_test.go b/prefetch_test.go index 4a134ab..7788d4d 100644 --- a/prefetch_test.go +++ b/prefetch_test.go @@ -11,24 +11,15 @@ import ( ) func TestPrefetchImages(t *testing.T) { - // single thread fp := "./prefetch" _ = os.Mkdir(fp, 0755) - prefetchImages("./pics", "./prefetch", "80") + prefetchImages("./pics/dir1/", "./prefetch") count := fileCount("./prefetch") - assert.Equal(t, 8, count) - _ = os.RemoveAll(fp) - - // concurrency - jobs = 2 - _ = os.Mkdir(fp, 0755) - prefetchImages("./pics", "./prefetch", "80") - count = fileCount("./prefetch") - assert.Equal(t, 6, count) + assert.Equal(t, int64(1), count) _ = os.RemoveAll(fp) } func TestBadPrefetch(t *testing.T) { jobs = 1 - prefetchImages("./pics2", "./prefetch", "80") + prefetchImages("./pics2", "./prefetch") } diff --git a/router.go b/router.go index e2e062b..773a1f3 100644 --- a/router.go +++ b/router.go @@ -3,15 +3,14 @@ package main import ( "errors" "fmt" - "github.com/gofiber/fiber/v2" - log "github.com/sirupsen/logrus" + "io/ioutil" "net/http" "net/url" "os" "path" - "path/filepath" - "strconv" - "strings" + + "github.com/gofiber/fiber/v2" + log "github.com/sirupsen/logrus" ) func convert(c *fiber.Ctx) error { @@ -24,14 +23,12 @@ func convert(c *fiber.Ctx) error { rawImageAbs = path.Join(config.ImgPath, reqURI) // /home/xxx/mypic/123.jpg } var imgFilename = path.Base(reqURI) // pure filename, 123.jpg - var finalFile string // We'll only need one c.sendFile() - var ua = c.Get("User-Agent") - var accept = c.Get("accept") - log.Debugf("Incoming connection from %s@%s with %s", ua, c.IP(), imgFilename) + log.Debugf("Incoming connection from %s %s", c.IP(), imgFilename) - needOrigin := goOrigin(accept, ua) - if needOrigin { - log.Debugf("A Safari/IE/whatever user has arrived...%s", ua) + goodFormat := guessSupportedFormat(&c.Request().Header) + + // old browser only, send the original image or fetch from remote and send. + if len(goodFormat) == 1 { c.Set("ETag", genEtag(rawImageAbs)) if proxyMode { localRemoteTmpPath := remoteRaw + reqURI @@ -42,20 +39,7 @@ func convert(c *fiber.Ctx) error { } } - // check ext - var allowed = false - for _, ext := range config.AllowedTypes { - haystack := strings.ToLower(imgFilename) - needle := strings.ToLower("." + ext) - if strings.HasSuffix(haystack, needle) { - allowed = true - break - } else { - allowed = false - } - } - - if !allowed { + if !checkAllowedType(imgFilename) { msg := "File extension not allowed! " + imgFilename log.Warn(msg) if imageExists(rawImageAbs) { @@ -81,45 +65,26 @@ func convert(c *fiber.Ctx) error { return errors.New(msg) } - _, webpAbsPath := genWebpAbs(rawImageAbs, config.ExhaustPath, imgFilename, reqURI) + // generate with timestamp to make sure files are update-to-date + avifAbs, webpAbs := genOptimizedAbs(rawImageAbs, config.ExhaustPath, imgFilename, reqURI) + convertFilter(rawImageAbs, avifAbs, webpAbs, nil) - if imageExists(webpAbsPath) { - finalFile = webpAbsPath - } else { - // we don't have abc.jpg.png1582558990.webp - // delete the old pic and convert a new one. - // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp - destHalfFile := path.Clean(path.Join(webpAbsPath, path.Dir(reqURI), imgFilename)) - matches, err := filepath.Glob(destHalfFile + "*") - if err != nil { - log.Error(err.Error()) - } else { - // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558100.webp <- older ones will be removed - // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp <- keep the latest one - for _, p := range matches { - if strings.Compare(destHalfFile, p) != 0 { - _ = os.Remove(p) - } - } + var availableFiles = []string{rawImageAbs} + for _, v := range goodFormat { + if "avif" == v { + availableFiles = append(availableFiles, avifAbs) } - - //for webp, we need to create dir first - err = os.MkdirAll(path.Dir(webpAbsPath), 0755) - q, _ := strconv.ParseFloat(config.Quality, 32) - err = webpEncoder(rawImageAbs, webpAbsPath, float32(q), true, nil) - - if err != nil { - log.Error(err) - _ = c.SendStatus(400) - _ = c.Send([]byte("Bad file. " + err.Error())) - return err + if "webp" == v { + availableFiles = append(availableFiles, webpAbs) } - finalFile = webpAbsPath } + + var finalFile = findSmallestFiles(availableFiles) etag := genEtag(finalFile) c.Set("ETag", etag) - c.Set("X-Compression-Rate", getCompressionRate(rawImageAbs, webpAbsPath)) - finalFile = chooseLocalSmallerFile(rawImageAbs, webpAbsPath) + c.Set("X-Compression-Rate", getCompressionRate(rawImageAbs, finalFile)) + buf, _ := ioutil.ReadFile(finalFile) + c.Set("content-type", getFileContentType(buf)) return c.SendFile(finalFile) } @@ -141,12 +106,8 @@ func proxyHandler(c *fiber.Ctx, reqURI string) error { cleanProxyCache(config.ExhaustPath + reqURI + "*") localRawImagePath := remoteRaw + reqURI _ = fetchRemoteImage(localRawImagePath, realRemoteAddr) - q, _ := strconv.ParseFloat(config.Quality, 32) _ = os.MkdirAll(path.Dir(localEtagWebPPath), 0755) - err := webpEncoder(localRawImagePath, localEtagWebPPath, float32(q), true, nil) - if err != nil { - log.Warning(err) - } + webpEncoder(localRawImagePath, localEtagWebPPath, config.Quality) chooseProxy(remoteLength, localEtagWebPPath) return c.SendFile(localEtagWebPPath) } diff --git a/router_test.go b/router_test.go index 10a3eea..f294555 100644 --- a/router_test.go +++ b/router_test.go @@ -15,8 +15,10 @@ import ( ) var ( - chromeUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36" - safariUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1 Safari/605.1.15" + chromeUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36" + acceptWebP = "image/webp,image/apng,image/*,*/*;q=0.8" + acceptLegacy = "image/jpeg" + safariUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1 Safari/605.1.15" ) func setupParam() { @@ -29,10 +31,14 @@ func setupParam() { remoteRaw = "remote-raw" } -func requestToServer(url string, app *fiber.App, ua string) (*http.Response, []byte) { +func requestToServer(url string, app *fiber.App, ua, accept string) (*http.Response, []byte) { req := httptest.NewRequest("GET", url, nil) req.Header.Set("User-Agent", ua) - resp, _ := app.Test(req, 60000) + req.Header.Set("Accept", accept) + resp, err := app.Test(req, 120000) + if err != nil { + return nil, nil + } data, _ := ioutil.ReadAll(resp.Body) return resp, data } @@ -44,7 +50,7 @@ func TestServerHeaders(t *testing.T) { url := "http://127.0.0.1:3333/webp_server.bmp" // test for chrome - response, _ := requestToServer(url, app, chromeUA) + response, _ := requestToServer(url, app, chromeUA, acceptWebP) ratio := response.Header.Get("X-Compression-Rate") etag := response.Header.Get("Etag") @@ -52,7 +58,7 @@ func TestServerHeaders(t *testing.T) { assert.NotEqual(t, "", etag) // test for safari - response, _ = requestToServer(url, app, safariUA) + response, _ = requestToServer(url, app, safariUA, acceptLegacy) ratio = response.Header.Get("X-Compression-Rate") etag = response.Header.Get("Etag") @@ -66,9 +72,9 @@ func TestConvert(t *testing.T) { "http://127.0.0.1:3333/webp_server.jpg": "image/webp", "http://127.0.0.1:3333/webp_server.bmp": "image/webp", "http://127.0.0.1:3333/webp_server.png": "image/webp", - "http://127.0.0.1:3333/empty.jpg": "text/plain; charset=utf-8", + "http://127.0.0.1:3333/empty.jpg": "", "http://127.0.0.1:3333/png.jpg": "image/webp", - "http://127.0.0.1:3333/12314.jpg": "text/plain; charset=utf-8", + "http://127.0.0.1:3333/12314.jpg": "", "http://127.0.0.1:3333/dir1/inside.jpg": "image/webp", "http://127.0.0.1:3333/%e5%a4%aa%e7%a5%9e%e5%95%a6.png": "image/webp", "http://127.0.0.1:3333/太神啦.png": "image/webp", @@ -78,9 +84,9 @@ func TestConvert(t *testing.T) { "http://127.0.0.1:3333/webp_server.jpg": "image/jpeg", "http://127.0.0.1:3333/webp_server.bmp": "image/bmp", "http://127.0.0.1:3333/webp_server.png": "image/png", - "http://127.0.0.1:3333/empty.jpg": "text/plain; charset=utf-8", + "http://127.0.0.1:3333/empty.jpg": "", "http://127.0.0.1:3333/png.jpg": "image/png", - "http://127.0.0.1:3333/12314.jpg": "text/plain; charset=utf-8", + "http://127.0.0.1:3333/12314.jpg": "", "http://127.0.0.1:3333/dir1/inside.jpg": "image/jpeg", } @@ -89,14 +95,14 @@ func TestConvert(t *testing.T) { // test Chrome for url, respType := range testChromeLink { - _, data := requestToServer(url, app, chromeUA) + _, data := requestToServer(url, app, chromeUA, acceptWebP) contentType := getFileContentType(data) assert.Equal(t, respType, contentType) } // test Safari for url, respType := range testSafariLink { - _, data := requestToServer(url, app, safariUA) + _, data := requestToServer(url, app, safariUA, acceptLegacy) contentType := getFileContentType(data) assert.Equal(t, respType, contentType) } @@ -112,13 +118,13 @@ func TestConvertNotAllowed(t *testing.T) { // not allowed, but we have the file url := "http://127.0.0.1:3333/webp_server.bmp" - _, data := requestToServer(url, app, chromeUA) + _, data := requestToServer(url, app, chromeUA, acceptWebP) contentType := getFileContentType(data) assert.Equal(t, "image/bmp", contentType) // not allowed, random file url = url + "hagdgd" - _, data = requestToServer(url, app, chromeUA) + _, data = requestToServer(url, app, chromeUA, acceptWebP) assert.Contains(t, string(data), "File extension not allowed") } @@ -132,7 +138,7 @@ func TestConvertProxyModeBad(t *testing.T) { // this is local random image, should be 500 url := "http://127.0.0.1:3333/webp_8888server.bmp" - resp, _ := requestToServer(url, app, chromeUA) + resp, _ := requestToServer(url, app, chromeUA, acceptWebP) assert.Equal(t, http.StatusInternalServerError, resp.StatusCode) } @@ -147,29 +153,26 @@ func TestConvertProxyModeWork(t *testing.T) { config.ImgPath = "https://webp.sh" url := "https://webp.sh/images/cover.jpg" - resp, data := requestToServer(url, app, chromeUA) + resp, data := requestToServer(url, app, chromeUA, acceptWebP) assert.Equal(t, http.StatusOK, resp.StatusCode) assert.Equal(t, "image/webp", getFileContentType(data)) // test proxyMode with Safari - resp, data = requestToServer(url, app, safariUA) + resp, data = requestToServer(url, app, safariUA, acceptLegacy) assert.Equal(t, http.StatusOK, resp.StatusCode) assert.Equal(t, "image/jpeg", getFileContentType(data)) } func TestConvertBigger(t *testing.T) { setupParam() - config.Quality = "100" - - jpg, _ := ioutil.ReadFile("pics/big.jpg") + config.Quality = 100 var app = fiber.New() app.Get("/*", convert) url := "http://127.0.0.1:3333/big.jpg" - response, data := requestToServer(url, app, chromeUA) + response, data := requestToServer(url, app, chromeUA, acceptWebP) assert.Equal(t, "image/jpeg", response.Header.Get("content-type")) - assert.True(t, len(data) == len(jpg)) - + assert.Equal(t, "image/jpeg", getFileContentType(data)) _ = os.RemoveAll(config.ExhaustPath) } diff --git a/update.go b/update.go index ee73861..ff2d623 100644 --- a/update.go +++ b/update.go @@ -3,13 +3,11 @@ package main import ( "encoding/json" "fmt" + log "github.com/sirupsen/logrus" + "github.com/staktrace/go-update" "io/ioutil" "net/http" - "os" - "path" "runtime" - - log "github.com/sirupsen/logrus" ) func autoUpdate() { @@ -47,12 +45,13 @@ func autoUpdate() { log.Debugf("%s-%s not found on release.", runtime.GOOS, runtime.GOARCH) return } - data, _ := ioutil.ReadAll(resp.Body) - _ = os.Mkdir("update", 0755) - err := ioutil.WriteFile(path.Join("update", filename), data, 0755) - if err == nil { - log.Info("Update complete. Please find your binary from update directory.") + err := update.Apply(resp.Body, update.Options{}) + if err != nil { + // error handling + log.Errorf("Update error. %v", err) + } else { + log.Info("Update complete. Please restart to apply changes.") } _ = resp.Body.Close() } diff --git a/update_test.go b/update_test.go index e757999..9a76d98 100644 --- a/update_test.go +++ b/update_test.go @@ -23,7 +23,7 @@ func Test404AutoUpdate(t *testing.T) { dir := "./update" releaseUrl = releaseUrl + "a" autoUpdate() - assert.Equal(t, 0, fileCount(dir)) + assert.Equal(t, int64(0), fileCount(dir)) _ = os.RemoveAll(dir) } diff --git a/webp-server.go b/webp-server.go index bbe9315..69de167 100644 --- a/webp-server.go +++ b/webp-server.go @@ -41,7 +41,7 @@ func deferInit() { FullTimestamp: true, TimestampFormat: "2006-01-02 15:04:05", CallerPrettyfier: func(f *runtime.Frame) (string, string) { - return fmt.Sprintf("[%s()]", f.Function), "" + return fmt.Sprintf("[%d:%s()]", f.Line, f.Function), "" }, } log.SetFormatter(Formatter) @@ -99,7 +99,7 @@ Develop by WebP Server team. https://github.com/webp-sh`, version) switchProxyMode() if prefetch { - go prefetchImages(config.ImgPath, config.ExhaustPath, config.Quality) + go prefetchImages(config.ImgPath, config.ExhaustPath) } app := fiber.New(fiber.Config{ diff --git a/webp-server_test.go b/webp-server_test.go index 4cb069e..6fb7bdc 100644 --- a/webp-server_test.go +++ b/webp-server_test.go @@ -21,7 +21,7 @@ func TestLoadConfig(t *testing.T) { assert.Equal(t, "./exhaust", c.ExhaustPath) assert.Equal(t, "127.0.0.1", c.Host) assert.Equal(t, "3333", c.Port) - assert.Equal(t, "80", c.Quality) + assert.Equal(t, float32(80), c.Quality) assert.Equal(t, "./pics", c.ImgPath) assert.Equal(t, []string{"jpg", "png", "jpeg", "bmp"}, c.AllowedTypes) } @@ -43,7 +43,7 @@ func TestMainFunction(t *testing.T) { // run main function go main() - time.Sleep(time.Second * 2) + time.Sleep(time.Second * 5) // verbose, prefetch assert.Equal(t, log.GetLevel(), log.DebugLevel) assert.True(t, verboseMode)