* recover middleware

* simplify Atoi

* metadata data prototype

* InterestingAttention

* resize itself

* Bump version to 0.9.4
Added some comments
Removed String() for Extraparams

* Add metadata test

* Fix CI

* Remove unnecessary tests

* Update file count

* use t.Run to get test case

---------

Co-authored-by: n0vad3v <n0vad3v@riseup.net>
This commit is contained in:
Benny 2023-07-11 19:08:32 +02:00 committed by GitHub
parent a5e3282ea1
commit a7b5992662
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 204 additions and 146 deletions

1
.gitignore vendored
View File

@ -25,3 +25,4 @@ remote-raw/
coverage.txt
.DS_Store
/webp_server_go
/metadata/*

View File

@ -3,7 +3,6 @@ package config
import (
"encoding/json"
"flag"
"fmt"
"os"
"regexp"
"runtime"
@ -59,10 +58,18 @@ var (
ProxyMode bool
Prefetch bool
Config jsonFile
Version = "0.9.3"
Version = "0.9.4"
WriteLock = cache.New(5*time.Minute, 10*time.Minute)
)
const Metadata = "metadata"
type MetaFile struct {
Id string `json:"id"` // hash of below path, also json file name id.webp
Path string `json:"path"` // local: path with width and height, proxy: full url
Checksum string `json:"checksum"` // hash of original file or hash(etag). Use this to identify changes
}
type jsonFile struct {
Host string `json:"HOST"`
Port string `json:"PORT"`
@ -81,7 +88,6 @@ func init() {
flag.BoolVar(&DumpConfig, "dump-config", false, "Print sample config.json")
flag.BoolVar(&DumpSystemd, "dump-systemd", false, "Print sample systemd service file.")
flag.BoolVar(&ShowVersion, "V", false, "Show version information.")
}
func LoadConfig() {
@ -100,11 +106,6 @@ type ExtraParams struct {
Height int // in px
}
// String : convert ExtraParams to string, used to generate cache path
func (e *ExtraParams) String() string {
return fmt.Sprintf("_width=%d&height=%d", e.Width, e.Height)
}
func switchProxyMode() {
matched, _ := regexp.MatchString(`^https?://`, Config.ImgPath)
if matched {

View File

@ -1,8 +1,9 @@
package config
import (
"github.com/stretchr/testify/assert"
"testing"
"github.com/stretchr/testify/assert"
)
func TestMain(m *testing.M) {
@ -21,15 +22,6 @@ func TestLoadConfig(t *testing.T) {
assert.Equal(t, Config.ExhaustPath, "./exhaust")
}
func TestExtraParamsString(t *testing.T) {
param := ExtraParams{
Width: 100,
Height: 100,
}
assert.Equal(t, param.String(), "_width=100&height=100")
}
func TestSwitchProxyMode(t *testing.T) {
switchProxyMode()
assert.False(t, ProxyMode)

View File

@ -4,9 +4,7 @@ import (
"errors"
"os"
"path"
"path/filepath"
"runtime"
"strings"
"sync"
"webp_server_go/config"
"webp_server_go/helper"
@ -32,7 +30,7 @@ func init() {
func resizeImage(img *vips.ImageRef, extraParams config.ExtraParams) error {
imgHeightWidthRatio := float32(img.Metadata().Height) / float32(img.Metadata().Width)
if extraParams.Width > 0 && extraParams.Height > 0 {
err := img.Thumbnail(extraParams.Width, extraParams.Height, 0)
err := img.Thumbnail(extraParams.Width, extraParams.Height, vips.InterestingAttention)
if err != nil {
return err
}
@ -85,27 +83,20 @@ func ConvertFilter(raw, avifPath, webpPath string, extraParams config.ExtraParam
}
}
func ResizeItself(raw, dest string, extraParams config.ExtraParams) {
log.Infof("Resize %s itself to %s", raw, dest)
img, _ := vips.LoadImageFromFile(raw, &vips.ImportParams{
FailOnError: boolFalse,
})
_ = resizeImage(img, extraParams)
buf, _, _ := img.ExportNative()
_ = os.WriteFile(dest, buf, 0600)
img.Close()
}
func convertImage(raw, optimized, imageType string, extraParams config.ExtraParams) error {
// we don't have /path/to/tsuki.jpg.1582558990.webp, maybe we have /path/to/tsuki.jpg.1082008000.webp
// delete the old converted pic and convert a new one.
// optimized: /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
// we'll delete file starts with /home/webp_server/exhaust/path/to/tsuki.jpg.ts.imageType
// If contain extraParams like tsuki.jpg?width=200, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=200
s := strings.Split(path.Base(optimized), ".")
pattern := path.Join(path.Dir(optimized), s[0]+"."+s[1]+".*."+s[len(s)-1])
matches, err := filepath.Glob(pattern)
if err != nil {
log.Error(err.Error())
} else {
for _, p := range matches {
_ = os.Remove(p)
}
}
// we need to create dir first
err = os.MkdirAll(path.Dir(optimized), 0755)
var err = os.MkdirAll(path.Dir(optimized), 0755)
if err != nil {
log.Error(err.Error())
}

View File

@ -5,7 +5,6 @@ import (
"os"
"path"
"path/filepath"
"strings"
"time"
"webp_server_go/config"
"webp_server_go/helper"
@ -35,8 +34,8 @@ func PrefetchImages() {
return nil
}
// RawImagePath string, ImgFilename string, reqURI string
proposedURI := strings.Replace(picAbsPath, config.Config.ImgPath, "", 1)
avif, webp := helper.GenOptimizedAbsPath(picAbsPath, proposedURI, config.ExtraParams{Width: 0, Height: 0})
metadata := helper.ReadMetadata(picAbsPath, "")
avif, webp := helper.GenOptimizedAbsPath(metadata)
_ = os.MkdirAll(path.Dir(avif), 0755)
log.Infof("Prefetching %s", picAbsPath)
go ConvertFilter(picAbsPath, avif, webp, config.ExtraParams{Width: 0, Height: 0}, finishChan)

View File

@ -75,28 +75,21 @@ func downloadFile(filepath string, url string) {
}
func fetchRemoteImg(url string) string {
func fetchRemoteImg(url string) config.MetaFile {
// url is https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200
// How do we know if the remote img is changed? we're using hash(url+etag) as key.
// if this exists in local system, means the remote img is not changed, we can use it directly.
// otherwise, we need to fetch it from remote and store it in local system.
// How do we know if the remote img is changed? we're using hash(etag+length)
log.Infof("Remote Addr is %s, pinging for info...", url)
// identifiable is etag + length
identifiable := pingURL(url)
// For store the remote raw image, /home/webp_server/remote-raw/3a42ab801f669d64-b8f999ab5acd69d03f5e904b1b84eb79210536
// Which 3a42ab801f669d64 is hash(url), b8f999ab5acd69d03f5e904b1b84eb79 is etag and 210536 is length
localRawImagePath := path.Join(config.RemoteRaw, helper.HashString(url)+"-"+identifiable)
etag := pingURL(url)
metadata := helper.ReadMetadata(url, etag)
localRawImagePath := path.Join(config.RemoteRaw, metadata.Id)
if helper.ImageExists(localRawImagePath) {
return localRawImagePath
} else {
// Temporary store of remote file.
cleanProxyCache(config.RemoteRaw + helper.HashString(url) + "*")
if !helper.ImageExists(localRawImagePath) || metadata.Checksum != helper.HashString(etag) {
// remote file has changed or local file not exists
log.Info("Remote file not found in remote-raw, re-fetching...")
cleanProxyCache(path.Join(config.Config.ExhaustPath, metadata.Id+"*"))
downloadFile(localRawImagePath, url)
return localRawImagePath
}
return metadata
}
func pingURL(url string) string {
@ -117,7 +110,5 @@ func pingURL(url string) string {
if etag == "" {
log.Info("Remote didn't return etag in header when getRemoteImageInfo, please check.")
}
// Remove " from etag
etag = strings.ReplaceAll(etag, "\"", "")
return etag + length
}

View File

@ -3,7 +3,6 @@ package handler
import (
"net/http"
"net/url"
"os"
"webp_server_go/config"
"webp_server_go/encoder"
"webp_server_go/helper"
@ -40,31 +39,42 @@ func Convert(c *fiber.Ctx) error {
reqURI = path.Clean(reqURI)
reqURIwithQuery = path.Clean(reqURIwithQuery)
WidthInt, err := strconv.Atoi(c.Query("width"))
if err != nil {
WidthInt = 0
}
HeightInt, err := strconv.Atoi(c.Query("height"))
if err != nil {
HeightInt = 0
}
width, _ := strconv.Atoi(c.Query("width"))
height, _ := strconv.Atoi(c.Query("height"))
var extraParams = config.ExtraParams{
Width: WidthInt,
Height: HeightInt,
Width: width,
Height: height,
}
var rawImageAbs string
var metadata = config.MetaFile{}
if config.ProxyMode {
// this is proxyMode, we'll have to use this url to download and save it to local path, which also gives us rawImageAbs
// https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200
rawImageAbs = fetchRemoteImg(config.Config.ImgPath + reqURIwithQuery)
metadata = fetchRemoteImg(config.Config.ImgPath + reqURIwithQuery)
rawImageAbs = path.Join(config.RemoteRaw, metadata.Id)
} else {
// not proxyMode, we'll use local path
rawImageAbs = path.Join(config.Config.ImgPath, reqURI) // /home/xxx/mypic/123.jpg
metadata = helper.ReadMetadata(reqURIwithQuery, "")
rawImageAbs = path.Join(config.Config.ImgPath, reqURI)
// detect if source file has changed
if metadata.Checksum != helper.HashFile(rawImageAbs) {
log.Info("Source file has changed, re-encoding...")
helper.WriteMetadata(reqURIwithQuery, "")
cleanProxyCache(path.Join(config.Config.ExhaustPath, metadata.Id))
}
}
goodFormat := helper.GuessSupportedFormat(&c.Request().Header)
// resize itself and return if only one format(raw) is supported
if len(goodFormat) == 1 {
dest := path.Join(config.Config.ExhaustPath, metadata.Id)
if !helper.ImageExists(dest) {
encoder.ResizeItself(rawImageAbs, dest, extraParams)
}
return c.SendFile(dest)
}
// Check the original image for existence,
if !helper.ImageExists(rawImageAbs) {
@ -75,11 +85,7 @@ func Convert(c *fiber.Ctx) error {
return nil
}
// generate with timestamp to make sure files are update-to-date
// If extraParams not enabled, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
// If extraParams enabled, and given request at tsuki.jpg?width=200, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=200&height=0
// If extraParams enabled, and given request at tsuki.jpg, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=0&height=0
avifAbs, webpAbs := helper.GenOptimizedAbsPath(rawImageAbs, reqURI, extraParams)
avifAbs, webpAbs := helper.GenOptimizedAbsPath(metadata)
encoder.ConvertFilter(rawImageAbs, avifAbs, webpAbs, extraParams, nil)
var availableFiles = []string{rawImageAbs}
@ -93,9 +99,7 @@ func Convert(c *fiber.Ctx) error {
}
finalFilename := helper.FindSmallestFiles(availableFiles)
buf, _ := os.ReadFile(finalFilename)
contentType := helper.GetFileContentType(buf)
contentType := helper.GetFileContentType(finalFilename)
c.Set("Content-Type", contentType)
c.Set("X-Compression-Rate", helper.GetCompressionRate(rawImageAbs, finalFilename))

View File

@ -1,7 +1,6 @@
package helper
import (
"bytes"
"fmt"
"os"
"path"
@ -10,34 +9,25 @@ import (
"time"
"webp_server_go/config"
"github.com/cespare/xxhash"
"github.com/h2non/filetype"
"github.com/cespare/xxhash"
"github.com/valyala/fasthttp"
log "github.com/sirupsen/logrus"
)
var _ = filetype.AddMatcher(filetype.NewType("avif", "image/avif"), avifMatcher)
func avifMatcher(buf []byte) bool {
// use hexdump on macOS to see the magic number
// 0000001c 66747970 61766966 00000000 61766966 6d696631 6d696166
magicHeader := []byte{
0x0, 0x0, 0x0, 0x1c,
0x66, 0x74, 0x79, 0x70,
0x61, 0x76, 0x69, 0x66,
0x0, 0x0, 0x0, 0x0,
0x61, 0x76, 0x69, 0x66,
0x6d, 0x69, 0x66, 0x31,
0x6d, 0x69, 0x61, 0x66,
func GetFileContentType(filename string) string {
if strings.HasSuffix(filename, ".webp") {
return "image/webp"
} else if strings.HasSuffix(filename, ".avif") {
return "image/avif"
} else {
// raw image, need to use filetype to determine
buf, _ := os.ReadFile(filename)
kind, _ := filetype.Match(buf)
return kind.MIME.Value
}
return len(buf) > 1 && bytes.Equal(buf[:28], magicHeader) || strings.Contains(string(buf), "ftypavif")
}
func GetFileContentType(buffer []byte) string {
kind, _ := filetype.Match(buffer)
return kind.MIME.Value
}
func FileCount(dir string) int64 {
@ -99,37 +89,11 @@ func CheckAllowedType(imgFilename string) bool {
return false
}
func GenOptimizedAbsPath(rawImagePath, reqURI string, extraParams config.ExtraParams) (string, string) {
// imageName is not needed, we can use reqURI
// get file mod time
var (
imageName = path.Base(reqURI)
exhaustPath = config.Config.ExhaustPath
)
STAT, err := os.Stat(rawImagePath)
if err != nil {
log.Error(err.Error())
return "", ""
}
ModifiedTime := STAT.ModTime().Unix()
// TODO: just hash it?
// webpFilename: abc.jpg.png -> abc.jpg.png.1582558990.webp
webpFilename := fmt.Sprintf("%s.%d.webp", imageName, ModifiedTime)
// avifFilename: abc.jpg.png -> abc.jpg.png.1582558990.avif
avifFilename := fmt.Sprintf("%s.%d.avif", imageName, ModifiedTime)
// If extraParams not enabled, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
// If extraParams enabled, and given request at tsuki.jpg?width=200, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=200&height=0
// If extraParams enabled, and given request at tsuki.jpg, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=0&height=0
if config.Config.EnableExtraParams {
webpFilename = webpFilename + extraParams.String()
avifFilename = avifFilename + extraParams.String()
}
// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
// Custom Exhaust: /path/to/exhaust/web_path/web_to/tsuki.jpg.1582558990.webp
webpAbsolutePath := path.Clean(path.Join(exhaustPath, path.Dir(reqURI), webpFilename))
avifAbsolutePath := path.Clean(path.Join(exhaustPath, path.Dir(reqURI), avifFilename))
func GenOptimizedAbsPath(metadata config.MetaFile) (string, string) {
webpFilename := fmt.Sprintf("%s.webp", metadata.Id)
avifFilename := fmt.Sprintf("%s.avif", metadata.Id)
webpAbsolutePath := path.Clean(path.Join(config.Config.ExhaustPath, webpFilename))
avifAbsolutePath := path.Clean(path.Join(config.Config.ExhaustPath, avifFilename))
return avifAbsolutePath, webpAbsolutePath
}
@ -214,3 +178,8 @@ func HashString(uri string) string {
// xxhash supports cross compile
return fmt.Sprintf("%x", xxhash.Sum64String(uri))
}
func HashFile(filepath string) string {
buf, _ := os.ReadFile(filepath)
return fmt.Sprintf("%x", xxhash.Sum64(buf))
}

View File

@ -18,7 +18,7 @@ func TestMain(m *testing.M) {
func TestFileCount(t *testing.T) {
// test helper dir
count := FileCount("./")
assert.Equal(t, int64(3), count)
assert.Equal(t, int64(4), count)
}
func TestImageExists(t *testing.T) {
@ -26,10 +26,6 @@ func TestImageExists(t *testing.T) {
assert.False(t, ImageExists("dgyuaikdsa"))
})
t.Run("file size incorrect", func(t *testing.T) {
assert.False(t, ImageExists("test.txt"))
})
// TODO: how to test lock?
t.Run("test dir", func(t *testing.T) {
@ -43,7 +39,7 @@ func TestImageExists(t *testing.T) {
func TestCheckAllowedType(t *testing.T) {
t.Run("not allowed type", func(t *testing.T) {
assert.False(t, CheckAllowedType("test.txt"))
assert.False(t, CheckAllowedType("./helper_test.go"))
})
t.Run("allowed type", func(t *testing.T) {

70
helper/metadata.go Normal file
View File

@ -0,0 +1,70 @@
package helper
import (
"encoding/json"
"net/url"
"os"
"path"
"webp_server_go/config"
log "github.com/sirupsen/logrus"
)
func getId(p string) (string, string, string) {
var id string
if config.ProxyMode {
return HashString(p), "", ""
}
parsed, _ := url.Parse(p)
width := parsed.Query().Get("width")
height := parsed.Query().Get("height")
// santizedPath will be /webp_server.jpg?width=200\u0026height= in local mode when requesting /webp_server.jpg?width=200
// santizedPath will be https://docs.webp.sh/images/webp_server.jpg?width=400 in proxy mode when requesting /images/webp_server.jpg?width=400 with IMG_PATH = https://docs.webp.sh
santizedPath := parsed.Path + "?width=" + width + "&height=" + height
id = HashString(santizedPath)
return id, path.Join(config.Config.ImgPath, parsed.Path), santizedPath
}
func ReadMetadata(p, etag string) config.MetaFile {
// try to read metadata, if we can't read, create one
var metadata config.MetaFile
var id, _, _ = getId(p)
buf, err := os.ReadFile(path.Join(config.Metadata, id+".json"))
if err != nil {
log.Warnf("can't read metadata: %s", err)
WriteMetadata(p, etag)
return ReadMetadata(p, etag)
}
err = json.Unmarshal(buf, &metadata)
if err != nil {
log.Warnf("unmarshal metadata error, possible corrupt file, re-building...: %s", err)
WriteMetadata(p, etag)
return ReadMetadata(p, etag)
}
return metadata
}
func WriteMetadata(p, etag string) config.MetaFile {
_ = os.Mkdir(config.Metadata, 0755)
var id, filepath, sant = getId(p)
var data = config.MetaFile{
Id: id,
}
if config.ProxyMode {
data.Path = p
data.Checksum = HashString(etag)
} else {
data.Path = sant
data.Checksum = HashFile(filepath)
}
buf, _ := json.Marshal(data)
_ = os.WriteFile(path.Join(config.Metadata, data.Id+".json"), buf, 0644)
return data
}

43
helper/metadata_test.go Normal file
View File

@ -0,0 +1,43 @@
package helper
import (
"net/url"
"path"
"testing"
"webp_server_go/config"
)
func TestGetId(t *testing.T) {
p := "https://example.com/image.jpg?width=200&height=300"
t.Run("proxy mode", func(t *testing.T) {
// Test case 1: Proxy mode
config.ProxyMode = true
id, jointPath, santizedPath := getId(p)
// Verify the return values
expectedId := HashString(p)
expectedPath := ""
expectedSantizedPath := ""
if id != expectedId || jointPath != expectedPath || santizedPath != expectedSantizedPath {
t.Errorf("Test case 1 failed: Expected (%s, %s, %s), but got (%s, %s, %s)",
expectedId, expectedPath, expectedSantizedPath, id, jointPath, santizedPath)
}
})
t.Run("non-proxy mode", func(t *testing.T) {
// Test case 2: Non-proxy mode
config.ProxyMode = false
p = "/image.jpg?width=400&height=500"
id, jointPath, santizedPath := getId(p)
// Verify the return values
parsed, _ := url.Parse(p)
expectedId := HashString(parsed.Path + "?width=400&height=500")
expectedPath := path.Join(config.Config.ImgPath, parsed.Path)
expectedSantizedPath := parsed.Path + "?width=400&height=500"
if id != expectedId || jointPath != expectedPath || santizedPath != expectedSantizedPath {
t.Errorf("Test case 2 failed: Expected (%s, %s, %s), but got (%s, %s, %s)",
expectedId, expectedPath, expectedSantizedPath, id, jointPath, santizedPath)
}
})
}

View File

@ -1 +0,0 @@
not an image

View File

@ -12,6 +12,7 @@ import (
"github.com/gofiber/fiber/v2"
"github.com/gofiber/fiber/v2/middleware/etag"
"github.com/gofiber/fiber/v2/middleware/logger"
"github.com/gofiber/fiber/v2/middleware/recover"
log "github.com/sirupsen/logrus"
)
@ -41,7 +42,8 @@ func setupLogger() {
Format: config.FiberLogFormat,
TimeFormat: config.TimeDateFormat,
}))
log.Infoln("Logger ready.")
app.Use(recover.New(recover.Config{}))
log.Infoln("fiber ready.")
}
func init() {
@ -59,7 +61,7 @@ func main() {
Webp Server Go - v%s
WebP Server Go - v%s
Develop by WebP Server team. https://github.com/webp-sh`, config.Version)
// process cli params
@ -88,7 +90,7 @@ Develop by WebP Server team. https://github.com/webp-sh`, config.Version)
app.Get("/*", handler.Convert)
fmt.Printf("\n %c[1;32m%s%c[0m\n\n", 0x1B, banner, 0x1B)
fmt.Println("Webp Server Go is Running on http://" + listenAddress)
fmt.Println("WebP Server Go is Running on http://" + listenAddress)
_ = app.Listen(listenAddress)